00010000:00000010:2.1F:1784945777.998004:0:337593:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000059c8c1eb. 00010000:00000010:3.1F:1784945777.999945:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000003607f751. 00010000:00000010:0.1F:1784945777.999951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000004ba52588. 00010000:00000010:3.1:1784945777.999956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000019c51290. 00010000:00000010:2.1:1784945778.000004:0:337544:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000ce729105. 00010000:00000010:2.1:1784945778.000010:0:337544:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000afab1df9. 00010000:00000010:2.1:1784945778.000013:0:337544:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000be9b2c0f. 00010000:00000010:1.1F:1784945778.001943:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000c18ca190. 00010000:00000010:1.1:1784945778.001956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006657dec3. 00010000:00000010:2.1:1784945778.026928:0:338490:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000c18ca190. 00010000:00000010:2.1:1784945778.027177:0:338490:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006657dec3. 00010000:00000010:2.1:1784945778.031922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000003c824233. 00010000:00000010:2.1:1784945778.031929:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000006aeba5f6. 00010000:00000010:1.1:1784945778.138613:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000623ce9e5. 00010000:00000010:1.1:1784945778.138626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000002661e20e. 00010000:00000010:0.1:1784945778.138949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006c2df6a1. 00010000:00000010:0.1:1784945778.138956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000017936e92. 00010000:00000010:0.1:1784945778.138959:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000145a0b6c. 00010000:00000010:1.1:1784945780.173969:0:338496:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000002cf34180. 00010000:00000010:1.1:1784945780.173975:0:338496:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000a3710301. 00010000:00000010:2.1:1784945780.174461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000003ebf868f. 00010000:00000010:2.1:1784945780.174468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000091d1e75d. 00010000:00000010:2.1:1784945780.179097:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000fb656dfb. 00010000:00000010:2.1:1784945780.179105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006657dec3. 00010000:00000010:2.1:1784945780.180942:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000145a0b6c. 00010000:00000010:2.1:1784945780.180949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000004e40ba27. 00010000:00000010:2.1:1784945780.183927:0:338496:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000002f4787ea. 00010000:00000010:2.1:1784945780.183937:0:338496:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000ce729105. 00010000:00000010:3.1:1784945780.183938:0:337708:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000003ebf868f. 00010000:00000010:3.1:1784945780.183945:0:337708:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000091d1e75d. 00010000:00000010:3.1:1784945780.193926:0:338746:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000050cb4045. 00010000:00000010:3.1:1784945780.193933:0:338746:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000d8d712d1. 00010000:00000010:3.1:1784945780.197927:0:338745:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000034d1bae6. 00010000:00000010:3.1:1784945780.197936:0:338745:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000085d5751e. 00010000:00000010:2.1:1784945780.199930:0:337708:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000001483bb6b. 00010000:00000010:2.1:1784945780.199936:0:337708:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000ce729105. 00010000:00000010:2.1:1784945780.199938:0:337708:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000000f9037d3. 00010000:00000010:2.1:1784945780.199941:0:337708:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006657dec3. 00010000:00000010:3.1:1784945780.200658:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000005f2f3d84. 00010000:00000010:3.1:1784945780.200663:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000049e5dc60. 00010000:00000010:2.1:1784945780.206929:0:338748:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000008cd806bc. 00010000:00000010:2.1:1784945780.206940:0:338748:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000d0976a63. 00010000:00000010:2.1:1784945780.206944:0:338748:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000046a27bd1. 00010000:00000010:2.1:1784945780.206946:0:338748:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000008b077f2e. 00010000:00000010:3.1:1784945780.210081:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000162566f9. 00010000:00000010:3.1:1784945780.210090:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000004820b7c6. 00010000:00000010:2.1:1784945780.212961:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000c18ca190. 00010000:00000010:2.1:1784945780.212970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000067d63661. 00010000:00000010:1.1:1784945780.222927:0:338739:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000002cce56a. 00010000:00000010:1.1:1784945780.222935:0:338739:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000118feb76. 00010000:00000010:1.1:1784945780.222942:0:338739:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000060e9d7c4. 00010000:00000010:1.1:1784945780.222947:0:338739:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000007e41c1d0. 00010000:00000010:2.1:1784945780.222964:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000009939501. 00010000:00000010:2.1:1784945780.222970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000c0cf9337. 00010000:00000010:2.1:1784945780.224924:0:338752:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000002cf34180. 00010000:00000010:2.1:1784945780.224932:0:338752:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000051e7ab43. 00010000:00000010:3.1:1784945780.225026:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000343e1f14. 00010000:00000010:3.1:1784945780.225032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000ba763059. 00010000:00000010:3.1:1784945780.225155:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000097e94f06. 00010000:00000010:3.1:1784945780.225160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000f1c4edcd. 00010000:00000010:2.1:1784945780.227925:0:338754:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000ecc84adf. 00010000:00000010:2.1:1784945780.227934:0:338754:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000005110fc28. 00010000:00000010:3.1:1784945780.230948:0:336644:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000cbf8d67c. 00010000:00000010:3.1:1784945780.230956:0:336644:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006a918c7d. 00010000:00000010:3.1:1784945780.232929:0:338513:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000e0e79287. 00010000:00000010:3.1:1784945780.232936:0:338513:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000f1c4edcd. 00010000:00000010:2.1:1784945780.234944:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000162566f9. 00010000:00000010:2.1:1784945780.234956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000ae5e6a19. 00010000:00000010:2.1:1784945780.238986:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000008cd806bc. 00010000:00000010:2.1:1784945780.238993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000038aea92f. 00010000:00000010:0.1:1784945780.245925:0:338496:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000002f4787ea. 00010000:00000010:0.1:1784945780.245931:0:338496:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000829b73a8. 00010000:00000010:2.1:1784945780.245981:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000e4cbf894. 00010000:00000010:2.1:1784945780.245989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000016ce5f24. 00010000:00000010:2.1:1784945780.249932:0:338759:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000d504a3e7. 00010000:00000010:2.1:1784945780.249942:0:338759:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000004e40ba27. 00010000:00000010:0.1:1784945780.249995:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000084dd107f. 00010000:00000010:0.1:1784945780.250000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000009111d320. 00010000:00000010:3.1:1784945780.250246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000932e884b. 00010000:00000010:3.1:1784945780.250254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000030610227. 00000800:00000200:3.1:1784945784.137047:0:337108:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (2) 00000800:00000200:3.1:1784945784.137086:0:337108:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.37@tcp 0000000099cd7b6c deleted 00000800:00000010:3.1:1784945784.137089:0:337108:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 0000000099cd7b6c. 00000800:00000200:3.1:1784945784.139395:0:337109:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (3) 00000800:00000200:3.1:1784945784.139565:0:337109:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.202.37@tcp 0000000099cd7b6c deleted 00000800:00000010:3.1:1784945784.139568:0:337109:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 0000000099cd7b6c. 00000800:00000200:0.1:1784945784.771050:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00010000:00000010:0.1:1784945785.355962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006b18f8bd. 00010000:00000010:0.1:1784945785.587270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000829b73a8. 00010000:00000010:0.1:1784945785.591976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000cc5a900a. 00010000:00000010:0.1:1784945785.591988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000039b09c35. 00010000:00000010:3.1:1784945785.595949:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000ecc84adf. 00010000:00000010:3.1:1784945785.595971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000009111d320. 00010000:00000010:0.1:1784945785.610959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000038aea92f. 00010000:00000010:2.1:1784945786.266933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000008b4ebc43. 00010000:00000010:1.1:1784945786.270988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000a6df25c6. 00010000:00000010:0.1:1784945786.274942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000003c824233. 00010000:00000010:2.1:1784945786.274948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000019c51290. 00010000:00000010:0.1:1784945786.274957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000059c8c1eb. 00010000:00000010:2.1:1784945786.274959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000002f3cb73. 00010000:00000010:2.1:1784945794.238942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000007dca0057. 00010000:00000010:3.1:1784945794.240945:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000039276c5e. 00010000:00000010:0.1:1784945794.240948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000016ce5f24. 00010000:00000010:3.1:1784945794.240960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000067361f0d. 00010000:00000010:0.1:1784945794.240961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000005b7304a8. 00010000:00000010:0.1:1784945794.240966:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000c02315d6. 00010000:00000010:0.1:1784945794.240969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000a59d8639. 00010000:00000010:0.1:1784945794.240972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000ae5e6a19. 00010000:00000010:0.1:1784945794.240974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000043f3ce95. 00010000:00000010:0.1:1784945794.240976:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000084dbb65. 00010000:00000010:0.1:1784945794.240979:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000001de352b6. 00010000:00000010:0.1:1784945794.240981:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000f5bf507d. 00010000:00000010:0.1:1784945794.240988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000dd67e91f. 00010000:00000010:3.1:1784945794.240990:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000007017d71c. 00010000:00000010:0.1:1784945794.240993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000ae99b839. 00010000:00000010:3.1:1784945794.240997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000fcbac289. 00010000:00000010:0.1:1784945794.312922:0:339511:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000004af7ef80. 00010000:00000010:0.1:1784945794.312930:0:339511:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000067d63661. 00010000:00000010:0.1:1784945794.312936:0:339511:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000293b9c8a. 00010000:00000010:0.1:1784945794.312941:0:339511:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000004ba52588. 00010000:00000010:1.1:1784945794.313389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000a5729734. 00010000:00000010:1.1:1784945794.313398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 00000000e5689109. 00010000:00000010:1.1:1784945794.313404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000002661e20e. 00010000:00000010:1.1:1784945794.313411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000006873f31e. 00010000:00000010:0.1:1784945828.283405:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 0000000046a27bd1. 00010000:00000010:0.1:1784945828.283419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000016ce5f24. 00010000:00000010:2.1:1784945828.294039:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at 000000004034cf6a. 00010000:00000010:2.1:1784945828.294048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000001de352b6. 00010000:00000010:0.1:1784945828.294136:0:342632:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000005e290cfc. 00000800:00000200:2.1:1784945834.535400:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784945838.516778:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784945884.788436:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784945884.788497:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784945902.005243:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784945928.629103:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784945944.180508:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784945944.180545:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784945944.180603:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784945980.852865:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784946003.572652:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784946003.572733:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784946003.572800:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:2.1:1784946003.572891:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784946058.676808:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784946062.964559:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784946062.964655:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784946062.964746:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784946062.964803:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784946062.964883:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784946084.276689:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784946122.356498:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784946122.356576:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784946122.356655:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784946122.356717:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784946122.356767:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784946162.100556:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946181.748594:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784946181.748719:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784946181.748758:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784946181.748814:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784946181.748865:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784946181.748955:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784946240.116434:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784946240.116522:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784946240.116576:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784946240.116653:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784946240.116696:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784946240.116843:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784946240.948749:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946300.532467:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784946300.532570:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784946300.532662:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784946300.532708:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784946300.532771:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784946300.532833:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784946300.532974:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784946318.772727:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946359.924455:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784946359.924586:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784946359.924667:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784946359.924892:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784946359.924953:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784946359.925001:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784946359.925043:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784946359.925137:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784946372.020989:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946372.020994:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 8, 7 secs later) 00000800:00000200:2.1:1784946397.620852:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784946419.316544:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784946419.316681:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784946419.316786:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784946419.316834:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784946419.316934:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784946419.316997:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784946419.317076:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784946419.317138:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784946419.317233:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784946450.869013:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946478.708422:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784946478.708546:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784946478.708635:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784946478.708703:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784946478.708749:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784946478.708871:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784946478.708966:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784946478.709023:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784946478.709077:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784946502.068506:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784946527.668669:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784946527.668675:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 16, 16 secs later) 00000800:00000200:2.1:1784946528.693550:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784946528.693556:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 16, 15 secs later) 00000800:00000200:2.1:1784946537.076494:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784946537.076596:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784946537.076668:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:2.1:1784946537.076823:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:2.1:1784946537.076949:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784946537.077006:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.1:1784946537.077064:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784946537.077144:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784946537.077210:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784946537.077262:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784946537.077298:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784946553.268602:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946597.492861:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784946597.492959:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784946597.493032:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784946597.493087:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784946597.493141:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784946597.493240:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784946597.493300:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784946597.493386:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784946597.493450:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784946597.493604:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784946597.493660:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784946597.493729:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784946605.493106:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784946656.885870:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784946656.886065:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784946656.886222:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784946656.886372:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784946656.886475:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784946656.886545:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784946656.886598:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784946656.886656:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784946656.886733:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784946656.886796:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784946656.886877:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784946656.886946:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784946656.886989:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784946657.717249:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784946683.316504:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784946683.316510:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 32, 31 secs later) 00000800:00000200:1.1:1784946684.340580:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946684.340585:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 32, 30 secs later) 00000800:00000200:3.1:1784946709.940469:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784946709.940474:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 32, 5 secs later) 00000800:00000200:2.1:1784946710.964443:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784946710.964449:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 32, 3 secs later) 00000800:00000200:2.1:1784946715.252445:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784946715.252593:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784946715.252702:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:2.1:1784946715.252797:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:2.1:1784946715.252980:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784946715.253058:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:2.1:1784946715.253110:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.1:1784946715.253178:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784946715.253224:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784946715.253285:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784946715.253320:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784946715.253372:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784946715.253427:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784946715.253469:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784946735.540513:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784946775.668450:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784946775.668554:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784946775.668660:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784946775.668778:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784946775.668881:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784946775.668947:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784946775.668982:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784946775.669040:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784946775.669115:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784946775.669154:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784946775.669198:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784946775.669235:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784946775.669286:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784946775.669326:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784946775.669370:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784946813.364614:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784946834.036473:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784946834.036576:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784946834.036683:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784946834.036831:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784946834.036969:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784946834.037009:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784946834.037054:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784946834.037087:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784946834.037126:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784946834.037198:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784946834.037255:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784946834.037314:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784946834.037351:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784946834.037412:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784946834.037468:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784946834.037525:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784946867.636937:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784946867.636943:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 60 secs later) 00000800:00000200:0.1:1784946892.212608:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784946892.212612:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 35 secs later) 00000800:00000200:1.1:1784946893.236647:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946893.236655:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 34 secs later) 00000800:00000200:0.1:1784946893.428680:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784946893.428851:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784946893.429057:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784946893.429127:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784946893.429182:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784946893.429238:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784946893.429307:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784946893.429414:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784946893.429518:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784946893.429593:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784946893.429717:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784946893.429784:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784946893.429886:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784946893.429983:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784946893.430098:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784946893.430181:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784946893.430265:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784946918.837099:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946918.837108:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 9 secs later) 00000800:00000200:1.1:1784946919.860689:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946919.860698:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 8 secs later) 00000800:00000200:1.1:1784946944.436643:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784946953.844467:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784946953.844601:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784946953.844743:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784946953.844944:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784946953.845069:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784946953.845145:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784946953.845238:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784946953.845304:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784946953.845380:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784946953.845470:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784946953.845526:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784946953.845597:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784946953.845689:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784946953.845764:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784946953.845849:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784946953.845953:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784946953.846000:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784946953.846042:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784947012.212975:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784947012.213036:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784947012.213096:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:2.1:1784947012.213158:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:2.1:1784947012.213200:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784947012.213255:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:2.1:1784947012.213315:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.1:1784947012.213353:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784947012.213384:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.1:1784947012.213467:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784947012.213522:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784947012.213572:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784947012.213644:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784947012.213718:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784947012.213785:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784947012.213842:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784947012.213939:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784947012.214015:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784947012.214058:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784947022.260666:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947072.628545:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784947072.628553:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784947072.628781:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784947072.628953:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784947072.629093:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784947072.629240:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784947072.629312:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784947072.629366:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784947072.629451:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784947072.629523:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784947072.629588:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784947072.629664:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784947072.629760:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784947072.629822:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784947072.629900:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784947072.630007:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784947072.630094:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784947072.630156:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784947072.630257:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784947072.630318:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784947072.630395:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784947100.084513:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947100.084520:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 36 secs later) 00000800:00000200:0.1:1784947101.108520:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947101.108527:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 35 secs later) 00000800:00000200:2.1:1784947126.708477:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947126.708484:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784947127.732558:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947127.732566:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 9 secs later) 00000800:00000200:0.1:1784947130.996833:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784947130.997111:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784947130.997118:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784947130.997353:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784947130.997500:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784947130.997636:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784947130.997742:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784947130.997810:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784947130.997870:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784947130.997967:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784947130.998041:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784947130.998108:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784947130.998188:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784947130.998276:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784947130.998357:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784947130.998418:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784947130.998532:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784947130.998611:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784947130.998701:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784947130.998902:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784947130.999022:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784947152.308528:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947190.388511:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784947190.388632:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784947190.388758:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784947190.388762:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947190.388998:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784947190.389121:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784947190.389240:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784947190.389355:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784947190.389446:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784947190.389507:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784947190.389596:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784947190.389661:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784947190.389735:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784947190.389812:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784947190.389894:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784947190.390000:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784947190.390095:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784947190.390168:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784947190.390256:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784947190.390327:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784947190.390412:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784947190.390492:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784947190.390570:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784947230.132535:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947250.804510:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784947250.804609:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784947250.804720:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784947250.804851:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784947250.804856:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784947250.805082:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784947250.805183:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784947250.805282:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784947250.805418:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784947250.805497:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784947250.805561:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784947250.805623:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784947250.805665:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784947250.805701:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784947250.805778:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784947250.805929:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784947250.805981:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784947250.806058:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784947250.806096:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784947250.806146:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784947250.806188:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784947250.806227:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784947250.806269:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784947250.806325:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784947282.356830:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947282.356836:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1784947283.381043:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947283.381056:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 00000800:00000200:3.1:1784947308.980899:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947308.980930:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 31 secs later) 00000800:00000200:3.1:1784947309.172636:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784947309.172643:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947309.172842:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784947309.173028:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784947309.173138:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784947309.173289:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784947309.173296:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947309.173473:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784947309.173548:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784947309.173607:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784947309.173691:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784947309.173755:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784947309.173820:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784947309.173970:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784947309.174061:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784947309.174126:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784947309.174230:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784947309.174296:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784947309.174360:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784947309.174443:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784947309.174505:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784947309.174586:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784947309.174646:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784947309.174725:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784947309.174798:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.1:1784947309.174882:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784947310.004541:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947310.004548:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 30 secs later) 00000800:00000200:0.1:1784947335.605278:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947335.605285:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 5 secs later) 00000800:00000200:0.1:1784947336.628724:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947336.628741:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 4 secs later) 00000800:00000200:1.1:1784947361.204617:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947368.564756:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784947368.564962:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784947368.564964:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784947368.565052:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784947368.565157:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784947368.565211:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784947368.565278:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784947368.565280:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784947368.565357:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784947368.565405:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784947368.565457:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784947368.565529:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784947368.565568:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784947368.565600:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784947368.565698:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784947368.565733:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784947368.565767:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784947368.565845:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784947368.565899:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784947368.565972:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784947368.566036:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784947368.566088:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784947368.566149:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784947368.566203:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:0.1:1784947368.566270:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784947368.566326:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:0.1:1784947368.566419:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784947428.980630:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784947428.980777:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784947428.980981:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:2.1:1784947428.981002:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 10 secs later) 00000800:00000200:2.1:1784947428.981175:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:2.1:1784947428.981319:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784947428.981450:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:2.1:1784947428.981575:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.1:1784947428.981579:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 10 secs later) 00000800:00000200:2.1:1784947428.981754:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784947428.981858:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.1:1784947428.981975:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784947428.982077:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784947428.982141:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784947428.982202:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784947428.982290:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784947428.982356:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784947428.982419:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784947428.982531:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784947428.982567:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784947428.982603:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784947428.982657:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784947428.982699:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784947428.982749:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784947428.982816:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784947428.982879:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784947428.982957:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784947428.983024:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784947428.983096:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784947439.028433:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947487.348468:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784947487.348579:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784947487.348669:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784947487.348779:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784947487.348782:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784947487.348962:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784947487.349120:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784947487.349206:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784947487.349330:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784947487.349333:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784947487.349527:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784947487.349616:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784947487.349726:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784947487.349870:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784947487.349924:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784947487.349983:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784947487.350041:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784947487.350084:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784947487.350119:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784947487.350189:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784947487.350243:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784947487.350295:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784947487.350349:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784947487.350402:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784947487.350446:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784947487.350517:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:3.1:1784947487.350559:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:3.1:1784947487.350609:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784947491.252647:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947491.252655:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1784947492.276985:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947492.276992:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 00000800:00000200:3.1:1784947517.876797:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947517.876805:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1784947518.901144:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784947518.901149:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 31 secs later) 00000800:00000200:2.1:1784947543.476648:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947543.476654:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1784947544.500733:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947544.500741:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 5 secs later) 00000800:00000200:0.1:1784947546.740651:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784947546.740657:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1784947546.740834:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784947546.740957:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784947546.741078:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784947546.741215:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784947546.741219:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1784947546.741378:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784947546.741417:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784947546.741468:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784947546.741600:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784947546.741602:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1784947546.741675:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784947546.741726:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784947546.741781:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784947546.741839:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784947546.741937:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784947546.741984:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784947546.742062:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784947546.742105:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784947546.742139:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784947546.742189:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784947546.742254:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784947546.742316:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784947546.742393:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784947546.742456:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:0.1:1784947546.742525:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784947546.742617:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:0.1:1784947546.742680:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.1:1784947546.742724:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:0.1:1784947546.742769:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:3.1:1784947569.076517:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947606.136929:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784947606.137093:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784947606.137098:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784947606.137236:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784947606.137411:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784947606.137559:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784947606.137740:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784947606.137746:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784947606.138004:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784947606.138165:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784947606.138392:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784947606.138473:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784947606.138475:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784947606.138559:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784947606.138597:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784947606.138635:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784947606.138692:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784947606.138735:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784947606.138767:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784947606.138819:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784947606.138872:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784947606.138965:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784947606.139048:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784947606.139088:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784947606.139197:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784947606.139289:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:0.1:1784947606.139351:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784947606.139425:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:0.1:1784947606.139489:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.1:1784947606.139564:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:0.1:1784947606.139607:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:0.1:1784947606.139658:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005c32a916] -> 12345-111.222.173.16@tcp (2) 00000800:00000200:3.1:1784947647.924633:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947665.529795:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784947665.529898:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784947665.530075:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784947665.530081:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947665.530235:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784947665.530341:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784947665.530477:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784947665.530614:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784947665.530618:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947665.530784:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784947665.530890:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784947665.531004:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784947665.531103:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784947665.531105:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947665.531161:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784947665.531194:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784947665.531227:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784947665.531303:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784947665.531343:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784947665.531383:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784947665.531452:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784947665.531516:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784947665.531561:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784947665.531623:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784947665.531667:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784947665.531723:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784947665.531801:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784947665.531847:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784947665.531960:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:3.1:1784947665.532014:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.1:1784947665.532099:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:3.1:1784947665.532148:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000065394e9e] -> 12345-111.222.142.16@tcp (2) 00000800:00000200:3.1:1784947699.124949:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947699.124959:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 59 secs later) 00000800:00000200:2.1:1784947700.148750:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947700.148763:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1784947725.748958:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947725.748965:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1784947725.943244:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784947725.943376:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784947725.943483:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:2.1:1784947725.943645:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:2.1:1784947725.943649:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 13 secs later) 00000800:00000200:2.1:1784947725.943787:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784947725.943863:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:2.1:1784947725.944005:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.1:1784947725.944140:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784947725.944146:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 13 secs later) 00000800:00000200:2.1:1784947725.944227:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.1:1784947725.944289:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784947725.944322:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784947725.944372:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784947725.944373:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 13 secs later) 00000800:00000200:2.1:1784947725.944430:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784947725.944470:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784947725.944501:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784947725.944585:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784947725.944630:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784947725.944665:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784947725.944736:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784947725.944781:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784947725.944818:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784947725.944889:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784947725.944960:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784947725.944997:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784947725.945051:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784947725.945103:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784947725.945156:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.1:1784947725.945197:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784947725.945273:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fb0da18e] -> 12345-111.222.173.17@tcp (2) 00000800:00000200:1.1:1784947726.772662:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784947726.772668:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1784947751.348746:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784947751.348751:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1784947752.373009:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947752.373019:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1784947777.972766:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784947784.312054:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784947784.312061:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947784.312250:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784947784.312401:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784947784.312574:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784947784.312581:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947784.312854:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784947784.312992:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784947784.313072:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784947784.313160:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784947784.313163:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947784.313269:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784947784.313319:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784947784.313374:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784947784.313459:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784947784.313461:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784947784.313518:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784947784.313553:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784947784.313588:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784947784.313639:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784947784.313685:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784947784.313723:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784947784.313772:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784947784.313836:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784947784.313887:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784947784.313967:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784947784.314012:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.1:1784947784.314071:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784947784.314138:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:3.1:1784947784.314189:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.1:1784947784.314248:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b4aa8942] -> 12345-111.222.142.17@tcp (2) 00000800:00000200:2.1:1784947843.703805:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784947843.703990:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784947843.703995:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 13 secs later) 00000800:00000200:2.1:1784947843.704188:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:2.1:1784947843.704292:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784947843.704416:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:2.1:1784947843.704422:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 13 secs later) 00000800:00000200:2.1:1784947843.704627:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.1:1784947843.704770:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784947843.704880:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.1:1784947843.705056:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784947843.705058:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 13 secs later) 00000800:00000200:2.1:1784947843.705150:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784947843.705218:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784947843.705276:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784947843.705378:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784947843.705380:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 13 secs later) 00000800:00000200:2.1:1784947843.705462:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784947843.705526:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784947843.705608:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784947843.705691:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784947843.705752:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784947843.705813:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784947843.705866:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784947843.705965:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784947843.706029:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784947843.706093:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784947843.706153:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784947843.706225:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784947843.706310:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784947843.706395:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784947843.706455:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784947843.706525:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c2d0a9b7] -> 12345-111.222.173.18@tcp (2) 00000800:00000200:2.1:1784947855.796877:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784947903.100349:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784947903.100577:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784947903.100864:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784947903.100896:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784947903.101452:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784947903.101663:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784947903.101829:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784947903.102024:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784947903.102029:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784947903.102266:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784947903.102327:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784947903.102393:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784947903.102488:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784947903.102491:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784947903.102585:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784947903.102661:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784947903.102784:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784947903.102941:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784947903.102946:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784947903.103179:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784947903.103239:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784947903.103297:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784947903.103394:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784947903.103513:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784947903.103571:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784947903.103734:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784947903.103833:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784947903.104111:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784947903.104227:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784947903.104305:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784947903.104427:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:1.1:1784947903.104522:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784947903.104608:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784947903.104716:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c61291f] -> 12345-111.222.142.18@tcp (2) 00000800:00000200:2.1:1784947908.021291:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947908.021298:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 59 secs later) 00000800:00000200:0.1:1784947909.044839:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947909.044850:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1784947934.644752:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947934.644758:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1784947935.668512:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784947935.668516:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1784947960.244760:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784947960.244776:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1784947961.269024:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784947961.269043:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1784947962.490796:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784947962.491118:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784947962.491368:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784947962.491653:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784947962.491672:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784947962.492136:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784947962.492408:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784947962.492645:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784947962.492647:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784947962.492781:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784947962.492858:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784947962.492981:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784947962.493108:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784947962.493111:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784947962.493233:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784947962.493388:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784947962.493465:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784947962.493598:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784947962.493602:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784947962.493795:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784947962.493873:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784947962.493985:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784947962.494086:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784947962.494180:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784947962.494307:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784947962.494409:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784947962.494508:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784947962.494713:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784947962.494819:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784947962.495011:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784947962.495142:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784947962.495307:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000e638d940] -> 12345-111.222.173.19@tcp (2) 00000800:00000200:0.1:1784947986.868494:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784948021.879338:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784948021.879347:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1784948021.879561:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784948021.879717:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784948021.879854:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784948021.880056:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784948021.880061:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1784948021.880210:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784948021.880278:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784948021.880352:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784948021.880458:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784948021.880461:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1784948021.880562:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784948021.880618:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784948021.880701:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784948021.880783:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784948021.880785:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1784948021.880865:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784948021.880968:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784948021.881030:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784948021.881133:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784948021.881135:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1784948021.881219:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784948021.881275:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784948021.881334:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784948021.881429:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784948021.881489:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784948021.881547:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784948021.881625:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784948021.881685:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.1:1784948021.881740:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784948021.881809:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:3.1:1784948021.881881:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:3.1:1784948021.881974:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.1:1784948021.882054:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:3.1:1784948021.882140:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000056d0ce26] -> 12345-111.222.142.19@tcp (2) 00000800:00000200:1.1:1784948038.068684:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784948081.268438:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784948081.268554:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784948081.268558:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784948081.268741:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784948081.268851:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784948081.268969:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784948081.269074:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784948081.269078:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784948081.269232:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784948081.269349:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784948081.269414:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784948081.269470:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784948081.269472:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784948081.269528:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784948081.269563:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784948081.269613:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784948081.269666:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784948081.269667:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784948081.269725:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784948081.269758:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784948081.269811:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784948081.269863:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784948081.269865:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 13 secs later) 00000800:00000200:0.1:1784948081.269944:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784948081.269989:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784948081.270040:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784948081.270090:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784948081.270137:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:0.1:1784948081.270170:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:0.1:1784948081.270222:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784948081.270266:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:0.1:1784948081.270324:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:0.1:1784948081.270403:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.1:1784948081.270441:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:0.1:1784948081.270474:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:0.1:1784948081.270525:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c8a64d17] -> 12345-111.222.173.20@tcp (2) 00000800:00000200:0.1:1784948090.292607:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784948116.917115:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784948116.917119:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 60 secs later) 00000800:00000200:0.1:1784948117.940570:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784948117.940575:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 59 secs later) 00000800:00000200:1.1:1784948140.664836:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784948140.664959:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784948140.665111:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784948140.665115:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784948140.665300:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784948140.665417:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784948140.665610:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784948140.665732:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784948140.665734:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784948140.665809:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784948140.665852:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784948140.665969:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784948140.665971:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784948140.666107:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784948140.666209:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784948140.666250:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784948140.666327:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784948140.666329:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784948140.666397:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784948140.666447:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784948140.666503:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784948140.666615:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784948140.666617:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1784948140.666680:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784948140.666732:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784948140.666789:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784948140.666893:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784948140.666988:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784948140.667043:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784948140.667170:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784948140.667221:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:1.1:1784948140.667267:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784948140.667386:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784948140.667443:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784948140.667514:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000ac9970b] -> 12345-111.222.142.20@tcp (2) 00000800:00000200:3.1:1784948142.516629:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784948142.516634:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 34 secs later) 00000800:00000200:1.1:1784948143.540552:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948143.540559:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1784948168.116517:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784948168.116520:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 8 secs later) 00000800:00000200:1.1:1784948169.140521:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948169.140528:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1784948194.740484:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948199.028492:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784948199.028650:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784948199.028799:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784948199.028803:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948199.028954:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784948199.029122:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784948199.029223:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784948199.029296:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784948199.029297:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948199.029357:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784948199.029425:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784948199.029463:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784948199.029513:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784948199.029514:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948199.029571:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784948199.029649:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784948199.029688:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784948199.029739:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784948199.029741:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948199.029798:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784948199.029863:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784948199.029897:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784948199.030002:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784948199.030003:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948199.030084:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784948199.030134:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784948199.030196:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784948199.030243:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784948199.030290:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784948199.030328:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784948199.030411:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:1.1:1784948199.030511:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784948199.030549:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784948199.030593:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784948199.030661:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a3246c63] -> 12345-111.222.173.21@tcp (2) 00000800:00000200:3.1:1784948259.444810:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784948259.444817:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948259.445190:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784948259.445380:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784948259.445728:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784948259.445739:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948259.446145:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784948259.446299:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784948259.446453:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784948259.446796:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784948259.446803:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948259.447038:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784948259.447144:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784948259.447203:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784948259.447279:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784948259.447281:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948259.447370:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784948259.447421:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784948259.447504:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784948259.447614:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784948259.447616:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948259.447757:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784948259.447852:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784948259.447992:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784948259.448125:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784948259.448128:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948259.448227:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784948259.448279:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784948259.448329:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784948259.448401:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.1:1784948259.448494:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784948259.448563:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:3.1:1784948259.448639:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:3.1:1784948259.448740:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.1:1784948259.448815:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:3.1:1784948259.448989:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000029c53218] -> 12345-111.222.142.21@tcp (2) 00000800:00000200:3.1:1784948272.564612:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948318.847177:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:1.1:1784948318.847423:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784948318.847431:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948318.847809:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.1:1784948318.848039:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:1.1:1784948318.848242:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784948318.848248:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948318.848511:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784948318.848592:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784948318.848687:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784948318.848841:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784948318.848844:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948318.849086:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784948318.849154:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784948318.849311:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784948318.849314:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948318.849481:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784948318.849625:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784948318.849721:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784948318.849846:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784948318.849850:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948318.850088:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784948318.850181:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784948318.850267:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784948318.850376:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784948318.850380:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784948318.850543:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784948318.850601:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784948318.850702:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784948318.850850:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784948318.851008:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:1.1:1784948318.851097:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784948318.851180:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784948318.851263:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784948318.851353:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002e3e75a5] -> 12345-111.222.173.22@tcp (2) 00000800:00000200:3.1:1784948325.812527:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784948325.812535:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 60 secs later) 00000800:00000200:1.1:1784948350.388934:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948350.388938:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 35 secs later) 00000800:00000200:1.1:1784948351.412519:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948351.412523:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 34 secs later) 00000800:00000200:2.1:1784948377.012806:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784948377.012817:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 8 secs later) 00000800:00000200:2.1:1784948377.204994:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:2.1:1784948377.205151:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:2.1:1784948377.205155:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948377.205356:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784948377.205488:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:2.1:1784948377.205693:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.1:1784948377.205697:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948377.205874:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784948377.205991:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.1:1784948377.206069:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784948377.206207:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784948377.206211:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948377.206460:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784948377.206599:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784948377.206696:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784948377.206830:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784948377.206833:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948377.206903:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784948377.207003:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784948377.207075:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784948377.207133:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784948377.207134:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948377.207216:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784948377.207268:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784948377.207326:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784948377.207404:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784948377.207407:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948377.207492:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784948377.207557:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784948377.207614:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784948377.207711:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784948377.207767:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.1:1784948377.207805:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784948377.207876:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784948377.207983:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051476c9c] -> 12345-111.222.142.22@tcp (2) 00000800:00000200:2.1:1784948378.036785:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784948378.036799:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1784948403.636533:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784948437.620789:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784948437.621045:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784948437.621255:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784948437.621514:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784948437.621523:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784948437.622023:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784948437.622227:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784948437.622359:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784948437.622362:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784948437.622498:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784948437.622598:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784948437.622723:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784948437.622857:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784948437.622860:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784948437.623004:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784948437.623096:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784948437.623170:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784948437.623279:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784948437.623283:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784948437.623393:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784948437.623470:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784948437.623546:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784948437.623680:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784948437.623684:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784948437.623812:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784948437.623890:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784948437.624005:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784948437.624112:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784948437.624114:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 13 secs later) 00000800:00000200:3.1:1784948437.624240:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.1:1784948437.624340:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784948437.624459:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:3.1:1784948437.624566:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:3.1:1784948437.624673:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.1:1784948437.624773:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:3.1:1784948437.624887:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002f52d4ca] -> 12345-111.222.173.23@tcp (2) 00000800:00000200:3.1:1784948481.460601:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784948495.989127:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784948495.989160:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948495.989449:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784948495.989660:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784948495.989869:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784948495.989878:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948495.990085:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784948495.990159:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784948495.990435:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784948495.990437:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948495.990691:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784948495.990837:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784948495.990953:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784948495.991048:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784948495.991051:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948495.991179:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784948495.991239:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784948495.991350:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784948495.991354:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948495.991485:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784948495.991551:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784948495.991605:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784948495.991693:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784948495.991695:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948495.991781:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784948495.991858:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:0.1:1784948495.991955:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:0.1:1784948495.992037:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784948495.992039:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.13 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948495.992128:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:0.1:1784948495.992246:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:0.1:1784948495.992330:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.1:1784948495.992416:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:0.1:1784948495.992482:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:0.1:1784948495.992544:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000537bbef7] -> 12345-111.222.142.23@tcp (2) 00000800:00000200:1.1:1784948534.708468:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948534.708474:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 60 secs later) 00000800:00000200:3.1:1784948556.405099:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:3.1:1784948556.405412:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:3.1:1784948556.405424:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 9 secs later) 00000800:00000200:3.1:1784948556.406235:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:3.1:1784948556.406435:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:3.1:1784948556.406675:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:3.1:1784948556.406944:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784948556.406951:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 9 secs later) 00000800:00000200:3.1:1784948556.407296:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784948556.407504:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784948556.407765:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:3.1:1784948556.407769:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 9 secs later) 00000800:00000200:3.1:1784948556.407917:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784948556.408089:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784948556.408159:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784948556.408249:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784948556.408252:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 9 secs later) 00000800:00000200:3.1:1784948556.408435:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784948556.408509:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784948556.408588:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784948556.408686:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784948556.408688:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 9 secs later) 00000800:00000200:3.1:1784948556.408777:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784948556.408832:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784948556.408880:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784948556.408999:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784948556.409000:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 9 secs later) 00000800:00000200:3.1:1784948556.409083:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784948556.409158:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784948556.409204:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.1:1784948556.409279:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784948556.409280:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.13 (cnted 0, interval 60, 25 secs later) 00000800:00000200:3.1:1784948556.409424:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:3.1:1784948556.409497:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:3.1:1784948556.409542:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.1:1784948556.409609:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:3.1:1784948556.409741:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000f65eb3b8] -> 12345-111.222.173.24@tcp (2) 00000800:00000200:2.1:1784948559.284601:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784948559.284624:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 35 secs later) 00000800:00000200:1.1:1784948560.308633:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948560.308638:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 34 secs later) 00000800:00000200:1.1:1784948585.909520:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948585.909534:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 9 secs later) 00000800:00000200:1.1:1784948586.933001:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948586.933011:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 8 secs later) 00000800:00000200:3.1:1784948611.509078:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784948615.810635:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:0.1:1784948615.810874:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784948615.811258:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784948615.811269:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948615.811631:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784948615.811826:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:0.1:1784948615.812200:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784948615.812213:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948615.812467:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784948615.812619:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784948615.812703:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784948615.812706:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948615.812891:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784948615.813047:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784948615.813137:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784948615.813319:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784948615.813322:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948615.813438:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784948615.813510:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784948615.813634:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784948615.813637:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948615.813761:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784948615.813952:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784948615.814040:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784948615.814127:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:0.1:1784948615.814129:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948615.814302:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:0.1:1784948615.814380:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784948615.814482:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:0.1:1784948615.814560:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:0.1:1784948615.814563:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.14 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784948615.814684:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.1:1784948615.814767:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:0.1:1784948615.814875:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:0.1:1784948615.815190:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000003f74a4] -> 12345-111.222.142.24@tcp (2) 00000100:00000001:0.0:1784948627.894749:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948627.894757:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948627.894766:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948627.894772:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948627.894778:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948627.894784:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948627.936031:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305567275, at 4305567276 00000100:00000010:0.0:1784948627.936075:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000afdabce. 00000100:00000001:0.0:1784948633.012507:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948633.012516:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948633.013255:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948633.013273:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d9530b6c. 00000400:00000200:0.0:1784948633.013287:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948633.013308:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948633.013320:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000400:00000010:0.0:1784948633.013327:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000100:00000001:0.0:1784948633.013345:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948633.013359:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000423173cd x1871651289641984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948649 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948633.013403:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948633.013416:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000423173cd x1871651289641984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948649 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948633.013454:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948633.013460:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1784948633.056045:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305572395, at 4305572396 00000100:00000010:0.0:1784948633.056071:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000040:0.0:1784948637.108507:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948637.108520:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948637.108530:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b6ab052. 00000020:00000040:0.0:1784948637.108541:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948637.108550:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948637.108557:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948637.108566:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948637.108577:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c70e9e66. 00000100:00000001:0.0:1784948637.108593:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948637.108609:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006b6ab052 x1871651289642112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948637.108631:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948637.108634:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948637.108645:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b6ab052] to pc [ptlrpcd_00_03+3] req@000000006b6ab052 x1871651289642112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948637.108681:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948637.108684:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948637.108689:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784948637.108693:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784948637.108696:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948637.108701:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784948637.108710:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10884 00000400:00000200:0.0:1784948637.108728:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784948637.108741:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 10883 00000400:00000200:0.0:1784948637.108759:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000040:0.0:1784948637.108772:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948637.108778:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948637.108786:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948637.108798:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000001:0.0:1784948637.108809:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948637.108813:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784948637.108819:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948637.108828:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:0.0:1784948637.108835:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784948637.108840:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:0.0:1784948637.108851:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948637.108859:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948637.108870:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948637.108878:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:0.0:1784948637.108886:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:0.0:1784948637.108900:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00080000:0.0:1784948637.109003:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:0.0:1784948637.109013:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948637.109024:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1784948637.109040:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948637.109053:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948637.109064:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784948637.109074:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948637.109088:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000038badef5. 00000020:00000040:0.0:1784948637.109099:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784948637.109114:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784948637.109123:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948637.109132:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948637.109137:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948637.109144:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948637.109162:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a868d044. 00000100:00000001:0.0:1784948637.109173:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948637.109193:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000038badef5 x1871651289642240/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948637.109233:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948637.109238:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948637.109251:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038badef5] to pc [ptlrpcd_rcv+-1] req@0000000038badef5 x1871651289642240/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948637.109291:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948637.109301:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948637.109307:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948637.109318:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000457a2a3. 00000020:00000040:0.0:1784948637.109328:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948637.109344:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948637.109354:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948637.109364:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948637.109385:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000076d8913a. 00000100:00000001:0.0:1784948637.109399:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948637.109419:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000000457a2a3 x1871651289642368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948637.109462:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948637.109470:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948637.109496:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000457a2a3] to pc [ptlrpcd_00_00+0] req@000000000457a2a3 x1871651289642368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948637.109596:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948637.109607:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10914) 00000100:00000001:0.0:1784948637.109653:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948637.109666:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948637.110460:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948637.110482:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948637.110495:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948637.110505:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948637.110532:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948637.110567:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6433580 00000400:00000200:0.0:1784948637.110595:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 145520 00000800:00000001:0.0:1784948637.110620:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948637.110630:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948637.110640:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948637.110659:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948637.110675:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948637.110681:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784948637.110692:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000c28fee1. 00000100:00000040:0.0:1784948637.110705:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000000c28fee1 x1871651289642368 msgsize 224 00000100:00100000:0.0:1784948637.110733:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948637.110768:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784948637.110785:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948637.110796:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948637.152094:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305576491, at 4305576492 00000100:00000010:0.0:1784948637.152117:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:0.0:1784948638.132330:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948638.132343:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948638.132355:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000457a2a3. 00000020:00000040:0.0:1784948638.132365:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948638.132378:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948638.132384:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948638.132392:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948638.132405:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b0d293c7. 00000100:00000001:0.0:1784948638.132416:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.132426:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000000457a2a3 x1871651289642496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948638.132441:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948638.132444:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.132461:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000457a2a3] to pc [ptlrpcd_00_01+1] req@000000000457a2a3 x1871651289642496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948638.132502:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948638.132506:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948638.132510:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784948638.132513:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784948638.132515:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948638.132519:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784948638.132525:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10884 00000400:00000200:0.0:1784948638.132538:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784948638.132544:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 10909 00000400:00000200:0.0:1784948638.132550:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:0.0:1784948638.132554:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1784948638.132560:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948638.132566:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948638.132575:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948638.132582:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:0.0:1784948638.132591:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948638.132598:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784948638.132604:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948638.132612:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:0.0:1784948638.132622:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784948638.132633:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:0.0:1784948638.132661:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948638.132668:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948638.132675:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948638.132683:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:0.0:1784948638.132691:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:0.0:1784948638.132701:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00080000:0.0:1784948638.132711:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:0.0:1784948638.132715:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948638.132721:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1784948638.132734:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948638.132744:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948638.132750:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784948638.132754:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948638.132760:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000058a1380c. 00000020:00000040:0.0:1784948638.132769:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784948638.132780:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784948638.132784:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948638.132790:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948638.132796:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948638.132801:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948638.132808:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006ac2bbf5. 00000100:00000001:0.0:1784948638.132816:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948638.132828:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000058a1380c x1871651289642624/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948638.132855:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948638.132858:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.132874:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058a1380c] to pc [ptlrpcd_rcv+-1] req@0000000058a1380c x1871651289642624/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948638.132990:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.132997:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948638.133007:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948638.133041:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e9bfc541. 00000020:00000040:0.0:1784948638.133054:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948638.133066:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948638.133073:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948638.133082:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948638.133092:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000faf98f5b. 00000100:00000001:0.0:1784948638.133102:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.133122:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e9bfc541 x1871651289642752/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948638.133157:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948638.133164:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.133183:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e9bfc541] to pc [ptlrpcd_00_02+2] req@00000000e9bfc541 x1871651289642752/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948638.133252:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.133266:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10915) 00000100:00000001:0.0:1784948638.133315:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948638.133326:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948638.133364:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948638.133374:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289642496 02000000:00000001:0.0:1784948638.133384:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948638.133392:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948638.133401:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948638.133415:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948638.133429:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289642496 00000020:00000001:0.0:1784948638.133436:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948638.133443:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784948638.133459:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948638.133468:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784948638.133480:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948638.133490:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948638.133507:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948638.133521:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:0.0:1784948638.133533:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948638.133546:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000020:00000010:0.0:1784948638.133566:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000015713dee. 00000020:00000010:0.0:1784948638.133583:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000db031200. 00000020:00000010:0.0:1784948638.133597:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948638.133613:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948638.133623:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948638.133630:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948638.133638:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948638.133659:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948638.133685:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948638.133712:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948638.133720:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948638.133740:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 670 00000100:00000040:0.0:1784948638.133765:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948638.133773:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742032464 : -115538967519152 : ffff96eafce22050) 00000100:00000040:0.0:1784948638.133789:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000004a69f61a x1871651289642496/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:299/0 lens 224/0 e 0 to 0 dl 1784948649 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948638.133839:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948638.133847:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:0.0:1784948638.133856:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948638.133864:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00100000:0.0:1784948638.133885:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000004a69f61a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289642496:12345-0@lo:400: 00000100:00000200:0.0:1784948638.133900:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289642496 00000020:00000001:0.0:1784948638.133977:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948638.133986:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948638.133993:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948638.133999:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948638.134005:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948638.134013:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948638.134018:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948638.134022:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948638.134025:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948638.134029:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948638.134035:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948638.134042:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948638.134052:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948638.134062:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e5f78452. 02000000:00000001:0.0:1784948638.134073:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948638.134080:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948638.134094:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948638.134108:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289642496 00010000:00000001:0.0:1784948638.134120:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948638.134133:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000004a69f61a x1871651289642496/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:299/0 lens 224/224 e 0 to 0 dl 1784948649 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948638.134193:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948638.134202:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948638.134213:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=110 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948638.134231:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948638.134242:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948638.134256:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948638.134267:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948638.134272:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948638.134279:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948638.134289:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948638.134296:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948638.134305:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289642496, offset 224 00000400:00000200:0.0:1784948638.134314:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948638.134327:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948638.134339:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6433600 00000400:00000200:0.0:1784948638.134347:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7dd5 [1] + 224 00000400:00000200:0.0:1784948638.134355:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948638.134361:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948638.134369:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948638.134374:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000457a2a3 x1871651289642496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948654 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948638.134387:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000457a2a3 x1871651289642496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948654 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948638.134417:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948638.134433:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784948638.134440:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784948638.134455:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948638.134461:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948638.134470:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948638.134483:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784948638.134493:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948638.134501:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948638.134511:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948638.134525:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004a69f61a x1871651289642496/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:299/0 lens 224/224 e 0 to 0 dl 1784948649 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948638.134553:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000004a69f61a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289642496:12345-0@lo:400: Request processed in 708us (1366us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948638.134590:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 670 00000100:00000040:0.0:1784948638.134602:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948638.134613:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948638.134622:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948638.134632:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000db031200. 00000020:00000010:0.0:1784948638.134684:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948638.134696:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000015713dee. 00000020:00000040:0.0:1784948638.134705:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:0.0:1784948638.134714:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784948638.134723:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000e5f78452. 02000000:00000001:0.0:1784948638.134733:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948638.134750:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948638.134759:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:0.0:1784948638.134774:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 00000400:00000010:0.0:1784948638.134787:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e65b0b8e. 00000100:00000040:0.0:1784948638.176043:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305577515, at 4305577516 00000100:00000010:0.0:1784948638.176058:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f68fd7be. 00000100:00000040:0.0:1784948643.252150:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948643.252199:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948643.252209:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001c81e7dd. 00000020:00000040:0.0:1784948643.252218:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948643.252224:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948643.252245:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948643.252263:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948643.252279:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000001adbd2d9. 00000100:00000001:0.0:1784948643.252304:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948643.252319:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000001c81e7dd x1871651289642880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948643.252347:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948643.252350:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948643.252367:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c81e7dd] to pc [ptlrpcd_00_03+3] req@000000001c81e7dd x1871651289642880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948643.252406:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948643.252409:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948643.252414:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948643.252419:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948643.252424:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948643.252428:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 00000020:00000040:0.0:1784948643.252432:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948643.252436:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948643.252443:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948643.252447:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948643.252456:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e471e641. 00000100:00000001:0.0:1784948643.252464:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948643.252481:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000f59e6b47 x1871651289643008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948643.252507:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948643.252512:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948643.252521:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59e6b47] to pc [ptlrpcd_00_00+0] req@00000000f59e6b47 x1871651289643008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948643.252570:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948643.252580:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10920) 00000100:00000001:0.0:1784948643.254614:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948643.254623:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289642880 02000000:00000001:0.0:1784948643.254636:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948643.254642:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948643.254646:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948643.254654:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948643.254665:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289642880 00000020:00000001:0.0:1784948643.254672:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948643.254677:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784948643.254687:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948643.254725:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784948643.254733:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948643.254743:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948643.254750:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948643.254773:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:0.0:1784948643.254781:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948643.254805:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000020:00000010:0.0:1784948643.254823:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000015713dee. 00000020:00000010:0.0:1784948643.254839:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a7c468cd. 00000020:00000010:0.0:1784948643.254855:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948643.254877:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948643.254887:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948643.254893:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948643.254900:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948643.254975:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948643.254996:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948643.255025:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948643.255031:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948643.255047:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 671 00000100:00000040:0.0:1784948643.255055:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948643.255061:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742036560 : -115538967515056 : ffff96eafce23050) 00000100:00000040:0.0:1784948643.255069:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000022e78e62 x1871651289642880/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:304/0 lens 224/0 e 0 to 0 dl 1784948654 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948643.255083:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948643.255088:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:0.0:1784948643.255091:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948643.255095:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000100:00100000:0.0:1784948643.255104:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000022e78e62 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289642880:12345-0@lo:400: 00000100:00000200:0.0:1784948643.255118:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289642880 00000020:00000001:0.0:1784948643.255127:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948643.255134:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948643.255139:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948643.255151:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948643.255170:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948643.255181:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948643.255188:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948643.255197:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948643.255205:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948643.255211:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948643.255221:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948643.255228:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948643.255238:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948643.255246:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007228af17. 02000000:00000001:0.0:1784948643.255265:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948643.255275:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948643.255284:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948643.255294:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289642880 00010000:00000001:0.0:1784948643.255303:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948643.255340:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000022e78e62 x1871651289642880/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:304/0 lens 224/224 e 0 to 0 dl 1784948654 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948643.255365:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948643.255368:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948643.255382:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=115 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948643.255388:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948643.255392:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948643.255396:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948643.255403:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948643.255405:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948643.255410:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948643.255417:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948643.255427:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948643.255434:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289642880, offset 224 00000400:00000200:0.0:1784948643.255441:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948643.255454:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948643.255466:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6433780 00000400:00000200:0.0:1784948643.255474:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7df9 [1] + 224 00000400:00000200:0.0:1784948643.255482:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948643.255490:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948643.255495:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948643.255506:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000001c81e7dd x1871651289642880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948659 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948643.255536:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000001c81e7dd x1871651289642880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948659 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948643.255579:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948643.255591:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784948643.255598:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784948643.255606:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948643.255613:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948643.255620:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948643.255629:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784948643.255635:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948643.255642:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948643.255649:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948643.255677:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000022e78e62 x1871651289642880/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:304/0 lens 224/224 e 0 to 0 dl 1784948654 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948643.255715:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000022e78e62 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289642880:12345-0@lo:400: Request processed in 624us (1178us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948643.255746:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 671 00000100:00000040:0.0:1784948643.255758:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948643.255768:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948643.255774:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948643.255783:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a7c468cd. 00000020:00000010:0.0:1784948643.255800:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948643.255811:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000015713dee. 00000020:00000040:0.0:1784948643.255825:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:0.0:1784948643.255837:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784948643.255844:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007228af17. 02000000:00000001:0.0:1784948643.255854:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948643.255862:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948643.255877:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:0.0:1784948643.255891:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000da8fa306. 00000400:00000010:0.0:1784948643.255903:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004ba703f8. 00000100:00000040:0.0:1784948643.297089:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305582636, at 4305582637 00000100:00000010:0.0:1784948643.297104:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000001:0.0:1784948648.372385:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948648.372394:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948648.372750:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948648.372758:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289643136 02000000:00000001:0.0:1784948648.372767:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948648.372773:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948648.372779:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948648.372807:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948648.372817:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289643136 00000020:00000001:0.0:1784948648.372824:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948648.372830:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784948648.372838:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948648.372846:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784948648.372853:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948648.372859:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948648.372865:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948648.372890:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:0.0:1784948648.372895:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948648.372899:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000020:00000010:0.0:1784948648.372939:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000015713dee. 00000020:00000010:0.0:1784948648.372949:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000001ad44e78. 00000020:00000010:0.0:1784948648.372959:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948648.372977:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948648.372984:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948648.372989:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948648.372995:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948648.373011:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948648.373030:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948648.373060:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948648.373065:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948648.373078:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 672 00000100:00000040:0.0:1784948648.373088:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948648.373093:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742029392 : -115538967522224 : ffff96eafce21450) 00000100:00000040:0.0:1784948648.373105:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000db799909 x1871651289643136/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:309/0 lens 224/0 e 0 to 0 dl 1784948659 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948648.373138:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948648.373146:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:0.0:1784948648.373155:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948648.373165:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00100000:0.0:1784948648.373179:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000db799909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289643136:12345-0@lo:400: 00000100:00000200:0.0:1784948648.373195:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289643136 00000020:00000001:0.0:1784948648.373203:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948648.373210:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948648.373217:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948648.373227:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948648.373250:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948648.373261:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948648.373267:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948648.373279:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948648.373286:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948648.373293:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948648.373298:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948648.373302:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948648.373306:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948648.373313:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000005c92614b. 02000000:00000001:0.0:1784948648.373320:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948648.373329:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948648.373333:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948648.373336:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289643136 00010000:00000001:0.0:1784948648.373343:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948648.373354:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000db799909 x1871651289643136/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:309/0 lens 224/224 e 0 to 0 dl 1784948659 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948648.373385:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948648.373388:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948648.373392:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=120 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948648.373399:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948648.373403:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948648.373408:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948648.373414:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948648.373418:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948648.373422:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948648.373425:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948648.373429:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948648.373452:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289643136, offset 224 00000400:00000200:0.0:1784948648.373465:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948648.373483:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948648.373496:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6433880 00000400:00000200:0.0:1784948648.373512:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7e15 [1] + 224 00000400:00000200:0.0:1784948648.373524:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948648.373533:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948648.373543:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948648.373551:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005e43a4d7 x1871651289643136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948664 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948648.373572:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005e43a4d7 x1871651289643136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948664 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948648.373620:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948648.373633:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784948648.373638:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784948648.373642:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948648.373648:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948648.373656:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948648.373664:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784948648.373675:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948648.373682:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948648.373689:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948648.373697:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db799909 x1871651289643136/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:309/0 lens 224/224 e 0 to 0 dl 1784948659 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948648.373722:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000db799909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289643136:12345-0@lo:400: Request processed in 580us (1086us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948648.373740:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 672 00000100:00000040:0.0:1784948648.373751:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948648.373760:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948648.373765:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948648.373775:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000001ad44e78. 00000020:00000010:0.0:1784948648.373787:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948648.373799:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000015713dee. 00000020:00000040:0.0:1784948648.373810:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:0.0:1784948648.373821:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784948648.373828:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000005c92614b. 02000000:00000001:0.0:1784948648.373840:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948648.373846:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948648.373859:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:0.0:1784948648.373891:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d30ef19f. 00000400:00000010:0.0:1784948648.373903:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005df8d94f. 00000100:00000040:0.0:1784948648.415018:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305587754, at 4305587755 00000100:00000010:0.0:1784948648.415038:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f99e6ef2. 00000100:00000001:0.0:1784948653.492313:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948653.492322:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948653.492567:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948653.492577:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948653.492962:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948653.492969:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289643392 02000000:00000001:0.0:1784948653.492976:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948653.492983:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948653.492988:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948653.492996:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948653.493009:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289643392 00000020:00000001:0.0:1784948653.493018:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948653.493024:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784948653.493031:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948653.493037:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784948653.493049:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948653.493060:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948653.493074:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948653.493086:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:0.0:1784948653.493096:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948653.493105:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000020:00000010:0.0:1784948653.493119:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000015713dee. 00000020:00000010:0.0:1784948653.493149:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006ba54408. 00000020:00000010:0.0:1784948653.493159:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948653.493184:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948653.493196:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948653.493203:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948653.493212:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948653.493229:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948653.493254:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948653.493281:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948653.493290:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948653.493305:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 673 00000100:00000040:0.0:1784948653.493319:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948653.493339:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742034512 : -115538967517104 : ffff96eafce22850) 00000100:00000040:0.0:1784948653.493355:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001a09b990 x1871651289643392/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:314/0 lens 224/0 e 0 to 0 dl 1784948664 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948653.493383:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948653.493407:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:0.0:1784948653.493416:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948653.493426:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00100000:0.0:1784948653.493443:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001a09b990 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289643392:12345-0@lo:400: 00000100:00000200:0.0:1784948653.493455:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289643392 00000020:00000001:0.0:1784948653.493462:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948653.493496:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948653.493500:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948653.493509:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948653.493512:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948653.493519:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948653.493521:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948653.493524:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948653.493528:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948653.493532:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948653.493537:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948653.493542:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948653.493548:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948653.493553:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000005c92614b. 02000000:00000001:0.0:1784948653.493558:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948653.493567:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948653.493573:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948653.493582:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289643392 00010000:00000001:0.0:1784948653.493588:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948653.493593:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001a09b990 x1871651289643392/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:314/0 lens 224/224 e 0 to 0 dl 1784948664 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948653.493602:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948653.493607:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948653.493611:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=125 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948653.493620:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948653.493628:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948653.493631:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948653.493634:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948653.493636:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948653.493640:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948653.493645:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948653.493650:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948653.493656:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289643392, offset 224 00000400:00000200:0.0:1784948653.493665:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948653.493673:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948653.493681:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6433980 00000400:00000200:0.0:1784948653.493691:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7e31 [1] + 224 00000400:00000200:0.0:1784948653.493701:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948653.493709:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948653.493715:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948653.493720:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000092889a66 x1871651289643392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948669 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948653.493738:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000092889a66 x1871651289643392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948669 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948653.493769:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948653.493778:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784948653.493783:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784948653.493790:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948653.493795:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948653.493803:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948653.493813:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784948653.493847:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948653.493857:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948653.493862:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948653.493872:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001a09b990 x1871651289643392/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:314/0 lens 224/224 e 0 to 0 dl 1784948664 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948653.493950:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001a09b990 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289643392:12345-0@lo:400: Request processed in 562us (1176us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948653.493970:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 673 00000100:00000040:0.0:1784948653.493981:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948653.493990:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948653.493999:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948653.494010:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006ba54408. 00000020:00000010:0.0:1784948653.494023:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948653.494035:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000015713dee. 00000020:00000040:0.0:1784948653.494053:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:0.0:1784948653.494066:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784948653.494072:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000005c92614b. 02000000:00000001:0.0:1784948653.494096:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948653.494104:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948653.494117:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:0.0:1784948653.494130:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e6551ee. 00000400:00000010:0.0:1784948653.494162:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b1caac62. 00000100:00000040:0.0:1784948653.535985:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305592875, at 4305592876 00000100:00000010:0.0:1784948653.536001:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b9b6859c. 00000100:00000001:0.0:1784948658.612240:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948658.612245:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.612279:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948658.612283:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948658.612286:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948658.612295:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948658.612323:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948658.612326:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948658.612331:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948658.612334:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948658.612338:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948658.612342:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000dc33132b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289643776:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948658.612350:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948658.612353:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948658.612354:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948658.612356:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948658.612359:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000015713dee. 02000000:00000001:0.0:1784948658.612363:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948658.612371:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000313d28ec. 00000400:00000010:0.0:1784948658.612376:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948658.612382:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289643776, portal 10 00000100:00000001:0.0:1784948658.612385:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948658.612388:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451662208 : -115544257889408 : ffff96e9c18d7b80) 00000100:00000040:0.0:1784948658.612398:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948658.612406:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948658.612409:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948658.612414:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00000100:00000200:0.0:1784948658.612419:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289643776, offset 0 00000400:00000200:0.0:1784948658.612432:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948658.612444:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948658.612450:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948658.612455:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948658.612458:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948658.612460:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948658.612463:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3065:3065:239:4294967295] 192.168.202.137@tcp LPNI seq info [2000:2000:8:4294967295] 00000400:00000200:0.0:1784948658.612480:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948658.612493:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948658.612502:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aae658b7. 00000800:00000200:0.0:1784948658.612517:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948658.612533:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948658.612540:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000aae658b7 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948658.612580:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948658.612590:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 1998 usecs (2 jiffies) 00000100:00000001:0.0:1784948658.612596:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305597954 00000100:00000001:0.0:1784948658.612610:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305597994 00000100:00000001:0.0:1784948658.612618:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305597954 00000100:00000001:0.0:1784948658.612640:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305597994 00000100:00000001:0.0:1784948658.612648:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305597954 00000100:00000001:0.0:1784948658.612664:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305597994 00000100:00000001:0.0:1784948658.612671:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305597954 00000100:00000001:0.0:1784948658.612683:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305597994 00000100:00000001:0.0:1784948658.612690:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948658.612695:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.612703:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.612711:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.612724:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948658.612728:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948658.612734:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948658.612740:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948658.612746:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.612753:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.612983:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948658.612987:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289643776 02000000:00000001:0.0:1784948658.612991:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948658.612994:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948658.612995:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948658.612999:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948658.613003:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289643776 00000020:00000001:0.0:1784948658.613007:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948658.613008:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948658.613009:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948658.613011:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948658.613013:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948658.613016:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948658.613018:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948658.613022:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:0.0:1784948658.613025:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948658.613029:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000020:00000010:0.0:1784948658.613036:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000020:00000010:0.0:1784948658.613042:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000923fc7d6. 00000020:00000010:0.0:1784948658.613048:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948658.613054:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948658.613058:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948658.613060:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948658.613063:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.613070:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948658.613074:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948658.613077:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.613105:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.613118:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948658.613123:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948658.613131:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 650 00000100:00000040:0.0:1784948658.613136:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948658.613141:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470025088 : -115544239526528 : ffff96e9c2a5ad80) 00000100:00000040:0.0:1784948658.613153:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005f9adc6b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:319/0 lens 224/0 e 0 to 0 dl 1784948669 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948658.613180:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948658.613183:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:0.0:1784948658.613186:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948658.613190:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00100000:0.0:1784948658.613197:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005f9adc6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289643776:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948658.613202:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289643776 00000020:00000001:0.0:1784948658.613205:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948658.613209:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948658.613211:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948658.613212:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948658.613216:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948658.613219:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948658.613223:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948658.613228:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948658.613230:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948658.613232:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948658.613235:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948658.613239:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948658.613241:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948658.613245:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e5f78452. 02000000:00000001:0.0:1784948658.613250:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.613254:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948658.613257:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948658.613259:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289643776 00010000:00000001:0.0:1784948658.613262:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948658.613267:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005f9adc6b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:319/0 lens 224/224 e 0 to 0 dl 1784948669 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948658.613280:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948658.613283:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948658.613286:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=125 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948658.613294:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948658.613298:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948658.613300:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948658.613302:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948658.613303:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.613305:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948658.613308:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948658.613312:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00000200:0.0:1784948658.613322:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289643776, offset 224 00000400:00000200:0.0:1784948658.613330:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948658.613341:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948658.613350:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3066:3066:239:4294967295] 192.168.202.137@tcp LPNI seq info [2001:2001:8:4294967295] 00000400:00000200:0.0:1784948658.613366:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948658.613377:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948658.613383:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c6406cdb. 00000800:00000200:0.0:1784948658.613392:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948658.613400:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948658.613405:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c6406cdb type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948658.613415:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948658.613423:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948658.613429:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948658.613431:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948658.613436:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948658.613443:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005f9adc6b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:319/0 lens 224/224 e 0 to 0 dl 1784948669 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948658.613463:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005f9adc6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289643776:12345-192.168.202.137@tcp:400: Request processed in 279us (518us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948658.613475:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 650 00000100:00000040:0.0:1784948658.613480:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948658.613487:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948658.613491:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948658.613495:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000923fc7d6. 00000020:00000010:0.0:1784948658.613502:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948658.613508:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000020:00000040:0.0:1784948658.613516:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948658.613522:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.613669:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948658.613672:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948658.613676:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948658.613679:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948658.613682:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000313d28ec. 00000400:00000200:0.0:1784948658.613685:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784948658.613689:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784948658.613692:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948658.613698:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948674 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948658.613708:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948674 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948658.613715:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948658.613716:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.613718:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948658.613719:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948658.613720:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948658.613721:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948658.613722:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948658.613725:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=125 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948658.613727:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=125 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948658.613729:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948658.613729:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948658.613731:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948658.613733:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948674 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948658.613738:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.613739:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948658.613740:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948658.613741:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948658.613742:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948658.613744:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948674 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948658.613748:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948658.613749:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948658.613750:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948658.613752:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948674 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948658.613756:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000dc33132b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289643776:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948658.613758:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.613759:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948658.613761:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000dc33132b x1871651289643776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948674 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948658.613769:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948658.613770:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948658.613771:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000015713dee. 02000000:00000001:0.0:1784948658.613774:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948658.613775:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948658.613775:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948658.613777:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948658.613778:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000062b4a223. 02000000:00000001:0.0:1784948658.613779:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948658.613780:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948658.613781:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000dc33132b. 00000100:00000001:0.0:1784948658.613797:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948658.613800:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.613801:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948658.613804:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948658.613805:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948658.613806:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948658.613807:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948658.655014:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305597994, at 4305597995 00000100:00000010:0.0:1784948658.655029:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e36e3817. 00000100:00000001:0.0:1784948663.732454:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948663.732466:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:0.0:1784948663.732477:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948663.732492:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948663.732500:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948663.732514:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948663.732521:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948663.732529:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948663.732546:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.732618:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948663.732624:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948663.732633:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948663.732639:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948663.732647:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948663.732663:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a1f38785 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289643904:0@lo:400: 00000100:00000001:0.0:1784948663.732677:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948663.732690:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948663.732697:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948663.732708:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948663.732733:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000015713dee. 02000000:00000001:0.0:1784948663.732765:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948663.732783:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000313d28ec. 00000400:00000010:0.0:1784948663.732799:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948663.732813:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289643904, portal 25 00000100:00000001:0.0:1784948663.732827:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948663.732834:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470028672 : -115544239522944 : ffff96e9c2a5bb80) 00000100:00000040:0.0:1784948663.732851:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.732877:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948663.732888:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948663.732896:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000200:0.0:1784948663.732948:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289643904, offset 0 00000400:00000200:0.0:1784948663.732964:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948663.732995:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948663.733028:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6433b80 00000400:00000200:0.0:1784948663.733048:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x74d5 [2] + 0 00000400:00000010:0.0:1784948663.733067:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000036dd0276. 00000400:00000200:0.0:1784948663.733078:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b0990f86 00000400:00000200:0.0:1784948663.733094:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948663.733112:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948663.733128:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0990f86 00000400:00000010:0.0:1784948663.733136:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b0990f86. 00000100:00000001:0.0:1784948663.733150:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948663.733157:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948663.733169:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000074ed294d x1871651289643904 msgsize 224 00000100:00100000:0.0:1784948663.733192:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948663.733201:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948663.733225:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948663.733300:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:0.0:1784948663.733308:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:0.0:1784948663.733318:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948663.733333:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.733396:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948663.733409:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.733451:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948663.733459:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948663.733471:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948663.733484:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305603115 00000100:00000010:0.0:1784948663.733499:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:0.0:1784948663.733514:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948663.733532:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305603115 00000100:00000010:0.0:1784948663.733546:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:0.0:1784948663.733560:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784948663.733572:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305603115 00000100:00000010:0.0:1784948663.733588:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:0.0:1784948663.733601:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948663.733611:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305603115 00000100:00000010:0.0:1784948663.733627:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:0.0:1784948663.733641:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948663.733654:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948663.733669:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948663.733681:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948663.733689:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948663.733885:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948663.733896:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000400:00000200:0.0:1784948663.733946:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948663.733966:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948663.733980:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007d821c01 00000400:00000010:0.0:1784948663.733988:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007d821c01. 00000100:00000001:0.0:1784948663.733998:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948663.734012:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fdc32c74 x1871651289644032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948679 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.734058:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948663.734074:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fdc32c74 x1871651289644032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948679 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.734091:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948663.734099:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948663.734338:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948663.734346:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948663.734354:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948663.734362:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948663.734370:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000313d28ec. 00000400:00000200:0.0:1784948663.734379:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784948663.734385:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784948663.734393:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948663.734405:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948663.734459:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.734479:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948663.734487:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948663.734496:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948663.734504:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948663.734509:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948663.734514:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948663.734517:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948663.734530:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=135 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948663.734543:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=135 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948663.734557:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948663.734561:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948663.734573:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948663.734579:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948663.734590:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948663.734604:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.734662:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948663.734668:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948663.734674:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948663.734682:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948663.734710:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a1f38785 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289643904:0@lo:400: 00000100:00000001:0.0:1784948663.734728:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948663.734734:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948663.734767:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a1f38785 x1871651289643904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948679 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948663.734794:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948663.734799:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948663.734804:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000015713dee. 02000000:00000001:0.0:1784948663.734816:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948663.734822:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948663.734828:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948663.734836:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948663.734842:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000006aefc2f9. 02000000:00000001:0.0:1784948663.734848:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948663.734854:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948663.734859:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a1f38785. 00000100:00000001:0.0:1784948663.734867:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948663.734874:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948663.734881:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948663.734891:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948663.734897:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948663.734942:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948663.734946:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948663.776012:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305603115, at 4305603116 00000100:00000010:0.0:1784948663.776029:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000001:0.0:1784948664.757343:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948664.757355:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.757435:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948664.757443:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948664.757454:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948664.757473:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004f710a78 x1871651289644416/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948664.757576:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948664.757585:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948664.757602:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948664.757611:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948664.757626:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948664.757647:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004f710a78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289644416:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784948664.757669:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948664.757679:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948664.757685:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948664.757694:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948664.757795:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ac2bbf5. 02000000:00000001:0.0:1784948664.757805:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948664.757820:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000313d28ec. 00000400:00000010:0.0:1784948664.757833:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948664.757843:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289644416, portal 4 00000100:00000001:0.0:1784948664.757855:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948664.757866:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451648768 : -115544257902848 : ffff96e9c18d4700) 00000100:00000040:0.0:1784948664.757891:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000004f710a78 x1871651289644416/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948664.757950:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948664.757965:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948664.757976:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000200:0.0:1784948664.757989:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289644416, offset 0 00000400:00000200:0.0:1784948664.758017:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948664.758044:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948664.758066:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948664.758085:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948664.758102:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948664.758115:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948664.758135:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3070:3070:238:4294967295] 192.168.202.137@tcp LPNI seq info [2005:2005:7:4294967295] 00000400:00000200:0.0:1784948664.758173:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948664.758197:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948664.758213:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002429e93e. 00000800:00000200:0.0:1784948664.758230:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948664.758242:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948664.758253:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002429e93e type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1784948664.758284:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948664.758293:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:0.0:1784948664.758302:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305604140 00000100:00000001:0.0:1784948664.758317:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305604139 00000100:00000001:0.0:1784948664.758332:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305604140 00000100:00000001:0.0:1784948664.758342:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305604139 00000100:00000001:0.0:1784948664.758354:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305604140 00000100:00000001:0.0:1784948664.758366:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305604139 00000100:00000001:0.0:1784948664.758380:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305604140 00000100:00000001:0.0:1784948664.758394:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305604139 00000100:00000001:0.0:1784948664.758407:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948664.758418:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.758428:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.758433:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948664.758442:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948664.758450:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1784948664.758461:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948664.758467:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948664.758476:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.758483:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948664.758683:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948664.758692:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002429e93e. 00000400:00000200:0.0:1784948664.758704:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948664.758724:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948664.758757:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:0.0:1784948664.758762:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:0.0:1784948664.758768:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948664.758775:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004f710a78 x1871651289644416/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948664.758793:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948664.758797:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000004f710a78 x1871651289644416/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948689 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948664.758880:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.758893:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948664.759004:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948664.759009:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289644544 02000000:00000001:0.0:1784948664.759015:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948664.759021:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948664.759025:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948664.759034:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948664.759043:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289644544 00000020:00000001:0.0:1784948664.759050:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948664.759075:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948664.759084:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948664.759092:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948664.759103:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948664.759112:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948664.759121:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948664.759130:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:0.0:1784948664.759140:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948664.759148:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000020:00000010:0.0:1784948664.759171:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000015713dee. 00000020:00000010:0.0:1784948664.759183:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ce2d3338. 00000020:00000010:0.0:1784948664.759191:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948664.759216:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948664.759225:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948664.759231:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948664.759238:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948664.759250:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948664.759256:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948664.759264:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.759284:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948664.759302:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948664.759308:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948664.759323:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 652 00000100:00000040:0.0:1784948664.759333:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948664.759339:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451646976 : -115544257904640 : ffff96e9c18d4000) 00000100:00000040:0.0:1784948664.759352:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000384a0555 x1871651289644544/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:325/0 lens 224/0 e 0 to 0 dl 1784948675 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948664.759378:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948664.759386:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:0.0:1784948664.759393:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948664.759398:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000100:00100000:0.0:1784948664.759410:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000384a0555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289644544:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948664.759421:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289644544 00000020:00000001:0.0:1784948664.759423:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948664.759426:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948664.759428:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948664.759431:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948664.759433:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948664.759437:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948664.759441:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948664.759443:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948664.759446:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948664.759448:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948664.759451:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948664.759455:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948664.759457:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948664.759460:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000005c92614b. 02000000:00000001:0.0:1784948664.759463:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.759466:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948664.759468:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948664.759471:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289644544 00010000:00000001:0.0:1784948664.759474:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948664.759478:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000384a0555 x1871651289644544/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:325/0 lens 224/224 e 0 to 0 dl 1784948675 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948664.759488:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948664.759490:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948664.759493:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=131 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948664.759498:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948664.759500:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948664.759503:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948664.759506:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948664.759508:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948664.759529:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948664.759533:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948664.759536:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000200:0.0:1784948664.759540:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289644544, offset 224 00000400:00000200:0.0:1784948664.759546:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948664.759555:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948664.759561:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3071:3071:239:4294967295] 192.168.202.137@tcp LPNI seq info [2006:2006:8:4294967295] 00000400:00000200:0.0:1784948664.759572:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948664.759580:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948664.759587:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002429e93e. 00000800:00000200:0.0:1784948664.759595:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948664.759625:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948664.759631:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002429e93e type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948664.759653:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948664.759659:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948664.759664:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948664.759667:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948664.759671:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948664.759679:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000384a0555 x1871651289644544/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:325/0 lens 224/224 e 0 to 0 dl 1784948675 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948664.759697:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000384a0555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289644544:12345-192.168.202.137@tcp:400: Request processed in 317us (1101us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948664.759707:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 652 00000100:00000040:0.0:1784948664.759711:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948664.759714:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948664.759717:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948664.759721:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ce2d3338. 00000020:00000010:0.0:1784948664.759726:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948664.759731:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000015713dee. 00000020:00000040:0.0:1784948664.759759:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948664.759763:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948664.759901:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948664.759941:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948664.759946:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948664.759953:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948664.759969:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948664.759985:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6433e00 00000400:00000200:0.0:1784948664.759995:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x7e8d [1] + 224 00000800:00000001:0.0:1784948664.760003:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948664.760009:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948664.760012:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948664.760017:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948664.760022:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948664.760027:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b55e20f2 x1871651289644544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948680 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948664.760038:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b55e20f2 x1871651289644544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948680 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948664.760060:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948664.760069:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948664.760072:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948664.801033:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305604140, at 4305604141 00000100:00000010:0.0:1784948664.801050:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000400:00000200:0.0:1784948665.972087:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948665.972093:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:0.0:1784948665.972099:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948665.972103:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948665.972107:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:0.0:1784948665.972110:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000400:00000200:0.0:1784948665.972114:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948665.972119:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:0.0:1784948665.972124:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef171f31. 00000400:00000200:0.0:1784948665.972132:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:0.0:1784948665.972137:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:0.0:1784948665.972143:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000400:00000200:0.0:1784948665.972150:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000400:00000001:0.0:1784948665.972155:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948665.972157:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948665.972160:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000400:00000010:0.0:1784948665.972164:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000400:00000200:0.0:1784948665.972168:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948665.972173:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:0.0:1784948665.972177:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e5260462. 00000400:00000200:0.0:1784948665.972183:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-110 00000400:00000200:0.0:1784948665.972188:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -110 00000400:00000200:0.0:1784948665.972193:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:0.0:1784948665.972197:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:0.0:1784948665.972199:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948665.972200:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948665.972201:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000400:00000010:0.0:1784948665.972202:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000400:00000200:0.0:1784948665.972204:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948665.972207:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:0.0:1784948665.972209:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ee879bbd. 00000400:00000200:0.0:1784948665.972213:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:0.0:1784948665.972214:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:0.0:1784948665.972216:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000400:00000200:0.0:1784948666.996171:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948666.996180:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:0.0:1784948666.996187:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948666.996189:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948666.996192:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:0.0:1784948666.996195:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000400:00000200:0.0:1784948666.996198:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948666.996202:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:0.0:1784948666.996206:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e7299d92. 00000400:00000200:0.0:1784948666.996212:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:0.0:1784948666.996214:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:0.0:1784948666.996217:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000800:00000001:0.0:1784948666.996332:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948666.996346:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948666.996351:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948666.996361:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948666.996375:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948666.996392:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948666.996402:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948666.996414:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948666.996418:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3073:3073:243:4294967295] 192.168.202.137@tcp LPNI seq info [2008:2008:8:4294967295] 00000400:00000200:0.0:1784948666.996426:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948666.996430:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948666.996444:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005e0a2c1e. 00000800:00000200:0.0:1784948666.996448:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948666.996453:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948666.996455:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005e0a2c1e type 3, nob 144 niov 1 nkiov 1 00000800:00000200:0.0:1784948666.996526:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948666.996531:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005e0a2c1e. 00000400:00000200:0.0:1784948666.996536:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948666.996542:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948668.020338:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948668.020352:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:0.0:1784948668.020363:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948668.020367:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948668.020371:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00000400:00000010:0.0:1784948668.020375:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000400:00000200:0.0:1784948668.020382:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948668.020389:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:0.0:1784948668.020396:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005152bcfe. 00000400:00000200:0.0:1784948668.020407:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:0.0:1784948668.020412:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:0.0:1784948668.020419:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00000800:00000200:0.0:1784948669.044235:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948669.044240:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009ff40a64. 00000400:00000200:0.0:1784948669.044246:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948669.044252:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948669.044257:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948669.044261:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784948669.044458:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948669.044467:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948669.044473:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948669.044476:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948669.044482:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948669.044491:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x7ead 00000800:00000001:0.0:1784948669.044496:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948669.044502:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948669.044503:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948669.044506:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948669.044510:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000400:00000010:0.0:1784948669.044512:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000400:00000200:0.0:1784948669.044514:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948669.044535:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a007df80. 00000400:00000010:0.0:1784948669.044538:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003bd24629. 00000800:00000001:0.0:1784948669.044544:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948669.044547:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948669.876081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948669.876099:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948669.876108:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948669.876119:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948669.876127:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948669.876133:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948669.876141:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948669.876292:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948669.876298:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948669.876304:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948669.876314:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b55e20f2 x1871651289644672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948669.876349:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948669.876355:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948669.876360:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948669.876364:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948669.876368:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948669.876373:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289644672:0@lo:400: 00000100:00000001:0.0:1784948669.876380:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948669.876388:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948669.876396:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948669.876404:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948669.876413:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000015713dee. 02000000:00000001:0.0:1784948669.876418:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948669.876428:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000968c2fa4. 00000400:00000010:0.0:1784948669.876446:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000100:00000200:0.0:1784948669.876460:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289644672, portal 25 00000100:00000001:0.0:1784948669.876467:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948669.876473:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451651456 : -115544257900160 : ffff96e9c18d5180) 00000100:00000040:0.0:1784948669.876486:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b55e20f2 x1871651289644672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948669.876528:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948669.876539:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948669.876548:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00000200:0.0:1784948669.876558:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289644672, offset 0 00000400:00000200:0.0:1784948669.876571:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948669.876601:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948669.876630:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6433e80 00000400:00000200:0.0:1784948669.876659:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7515 [2] + 0 00000400:00000010:0.0:1784948669.876672:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a9a70155. 00000400:00000200:0.0:1784948669.876683:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006c7a4b68 00000400:00000200:0.0:1784948669.876717:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948669.876731:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948669.876790:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006c7a4b68 00000400:00000010:0.0:1784948669.876800:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000006c7a4b68. 00000100:00000001:0.0:1784948669.876815:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948669.876821:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948669.876833:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000000c7e43f4 x1871651289644672 msgsize 224 00000100:00100000:0.0:1784948669.876847:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948669.876851:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948669.876865:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948669.876880:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:0.0:1784948669.876888:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:0.0:1784948669.876901:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948669.876953:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b55e20f2 x1871651289644672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948669.877000:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948669.877014:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b55e20f2 x1871651289644672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948685 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948669.877036:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948669.877044:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948669.877058:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948669.877069:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305609259 00000100:00000040:3.0:1784948669.877203:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948669.877213:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741514832 : -115539968036784 : ffff96eac13f7050) 00000100:00000001:0.0:1784948669.877217:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305609219 00000100:00000040:3.0:1784948669.877230:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000c7e43f4 x1871651289644672/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:330/0 lens 224/0 e 0 to 0 dl 1784948680 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948669.877233:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305609259 00000100:00000001:0.0:1784948669.877246:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305609220 00000100:00000001:0.0:1784948669.877259:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305609259 00000100:00000001:0.0:1784948669.877273:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305609220 00000100:00000001:3.0:1784948669.877275:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948669.877284:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305609259 00000020:00000010:3.0:1784948669.877287:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000056fd134f. 00000100:00000001:0.0:1784948669.877291:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305609220 00000100:00000001:3.0:1784948669.877293:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948669.877300:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000056fd134f. 00000100:00000001:0.0:1784948669.877302:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948669.877308:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948669.877314:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000c7e43f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289644672:12345-0@lo:400: 00000100:00000001:0.0:1784948669.877317:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948669.877324:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289644672 00000100:00000001:0.0:1784948669.877328:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948669.877330:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948669.877340:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784948669.877341:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948669.877347:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:3.0:1784948669.877347:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948669.877358:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784948669.877359:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948669.877362:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784948669.877365:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784948669.877372:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784948669.877374:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948669.877376:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948669.877380:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948669.877382:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948669.877390:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948669.877396:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948669.877401:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948669.877406:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948669.877413:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948669.877419:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 02000000:00000001:3.0:1784948669.877425:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948669.877428:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948669.877435:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948669.877442:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289644672 00010000:00000001:3.0:1784948669.877449:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948669.877457:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000c7e43f4 x1871651289644672/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:330/0 lens 224/224 e 0 to 0 dl 1784948680 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948669.877476:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948669.877481:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948669.877487:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=141 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948669.877497:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948669.877506:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948669.877510:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948669.877514:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948669.877517:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948669.877524:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948669.877533:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948669.877540:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000200:3.0:1784948669.877548:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289644672, offset 224 00000400:00000200:3.0:1784948669.877560:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948669.877561:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948669.877568:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289644800 02000000:00000001:0.0:1784948669.877573:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948669.877578:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:3.0:1784948669.877583:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948669.877584:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948669.877592:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948669.877598:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289644800 00000020:00000001:0.0:1784948669.877602:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:3.0:1784948669.877603:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6433e80 00000020:00000040:0.0:1784948669.877605:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948669.877612:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948669.877618:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:3.0:1784948669.877618:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7eb1 [1] + 224 00000020:00000001:0.0:1784948669.877628:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784948669.877632:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784948669.877633:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948669.877641:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784948669.877642:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:0.0:1784948669.877650:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:3.0:1784948669.877651:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948669.877658:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784948669.877659:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b55e20f2 x1871651289644672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948685 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948669.877684:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000020:00000010:0.0:1784948669.877721:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000100:00000040:3.0:1784948669.877747:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b55e20f2 x1871651289644672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948685 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948669.877749:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ce2d3338. 00000020:00000010:0.0:1784948669.877758:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948669.877782:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948669.877782:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948669.877788:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948669.877790:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:3.0:1784948669.877793:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000100:00000001:0.0:1784948669.877794:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948669.877804:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:0.0:1784948669.877806:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948669.877813:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948669.877815:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948669.877816:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948669.877824:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948669.877837:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948669.877837:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948669.877852:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948669.877858:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948669.877863:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000001:3.0:1784948669.877863:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948669.877874:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948669.877882:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 653 00000020:00000001:3.0:1784948669.877884:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948669.877892:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000040:3.0:1784948669.877897:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c7e43f4 x1871651289644672/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:330/0 lens 224/224 e 0 to 0 dl 1784948680 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948669.877901:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451654144 : -115544257897472 : ffff96e9c18d5c00) 00000100:00000040:0.0:1784948669.877979:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000004ddb289d x1871651289644800/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:330/0 lens 224/0 e 0 to 0 dl 1784948680 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948669.878019:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000c7e43f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289644672:12345-0@lo:400: Request processed in 734us (1176us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948669.878042:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:3.0:1784948669.878052:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 677 00000020:00000010:0.0:1784948669.878054:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000040:3.0:1784948669.878062:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948669.878066:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948669.878070:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948669.878075:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948669.878082:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000020:00000010:3.0:1784948669.878086:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000079fab95b. 00000100:00100000:0.0:1784948669.878104:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000004ddb289d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289644800:12345-192.168.202.137@tcp:400: 00000020:00000010:3.0:1784948669.878107:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000e923a07a. 00000020:00000010:3.0:1784948669.878118:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000100:00000200:0.0:1784948669.878129:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289644800 00000020:00000040:3.0:1784948669.878131:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:0.0:1784948669.878142:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:3.0:1784948669.878144:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948669.878152:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 00000020:00000001:0.0:1784948669.878154:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948669.878179:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948669.878180:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948669.878187:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948669.878188:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948669.878193:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000200:3.0:1784948669.878199:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:0.0:1784948669.878203:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:3.0:1784948669.878209:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e8bdea01. 00000020:00000001:0.0:1784948669.878210:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948669.878219:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000010:3.0:1784948669.878223:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c9198cec. 00000020:00000001:0.0:1784948669.878224:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948669.878231:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948669.878236:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948669.878242:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948669.878247:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948669.878257:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007228af17. 02000000:00000001:0.0:1784948669.878264:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948669.878268:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948669.878270:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948669.878278:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289644800 00010000:00000001:0.0:1784948669.878283:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948669.878292:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000004ddb289d x1871651289644800/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:330/0 lens 224/224 e 0 to 0 dl 1784948680 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948669.878321:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948669.878328:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948669.878338:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=136 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948669.878350:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948669.878362:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948669.878371:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948669.878383:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948669.878388:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948669.878401:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948669.878412:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948669.878419:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00000200:0.0:1784948669.878427:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289644800, offset 224 00000400:00000200:0.0:1784948669.878436:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948669.878447:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948669.878455:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3077:3077:244:4294967295] 192.168.202.137@tcp LPNI seq info [2012:2012:8:4294967295] 00000400:00000200:0.0:1784948669.878470:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948669.878478:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948669.878483:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005e0a2c1e. 00000800:00000200:0.0:1784948669.878488:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948669.878494:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948669.878498:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005e0a2c1e type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948669.878516:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948669.878519:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948669.878522:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948669.878524:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948669.878526:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948669.878531:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004ddb289d x1871651289644800/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:330/0 lens 224/224 e 0 to 0 dl 1784948680 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948669.878542:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000004ddb289d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289644800:12345-192.168.202.137@tcp:400: Request processed in 501us (1061us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948669.878552:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 653 00000100:00000040:0.0:1784948669.878555:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948669.878558:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948669.878560:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948669.878564:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ce2d3338. 00000020:00000010:0.0:1784948669.878568:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948669.878572:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000020:00000040:0.0:1784948669.878578:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948669.878581:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948669.878672:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948669.878679:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005e0a2c1e. 00000400:00000200:3.0:1784948669.878686:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948669.878721:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948669.878728:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:3.0:1784948669.878732:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:3.0:1784948669.878739:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948669.878743:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948669.878752:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948669.878762:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948669.878767:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948669.878772:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948669.878780:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948669.878793:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6433f00 00000400:00000200:3.0:1784948669.878802:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x7eb9 [1] + 224 00000800:00000001:3.0:1784948669.878809:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948669.878819:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948669.878823:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948669.878830:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948669.878836:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948669.878842:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000007fef3945 x1871651289644800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948685 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948669.878858:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000007fef3945 x1871651289644800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948685 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948669.878885:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948669.878893:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948669.878898:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948669.920050:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305609259, at 4305609260 00000100:00000040:0.0:1784948669.920057:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305609259, at 4305609260 00000100:00000010:3.0:1784948669.920072:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003b6d02be. 00000100:00000010:0.0:1784948669.920076:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f99e6ef2. 00000400:00000200:0.0:1784948670.068101:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948670.068113:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:0.0:1784948670.068119:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948670.068121:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948670.068123:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000400:00000010:0.0:1784948670.068125:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000400:00000200:0.0:1784948670.068129:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948670.068133:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:0.0:1784948670.068137:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e120de1c. 00000400:00000200:0.0:1784948670.068141:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000200:0.0:1784948670.068143:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:0.0:1784948670.068146:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:0.0:1784948670.068149:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000001:0.0:1784948670.068150:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948670.068151:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948670.068152:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000400:00000010:0.0:1784948670.068153:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000400:00000200:0.0:1784948670.068154:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948670.068156:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:0.0:1784948670.068157:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b8810b40. 00000400:00000200:0.0:1784948670.068159:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:0.0:1784948670.068160:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:0.0:1784948670.068162:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000100:00000001:3.0:1784948670.900003:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948670.900032:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948670.900035:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948670.900040:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948670.900043:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948670.900047:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948670.900049:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948671.092137:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948671.092145:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:0.0:1784948671.092149:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948671.092151:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948671.092153:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:0.0:1784948671.092155:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000400:00000200:0.0:1784948671.092159:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948671.092162:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:0.0:1784948671.092165:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089e37819. 00000400:00000200:0.0:1784948671.092169:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:0.0:1784948671.092172:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:0.0:1784948671.092175:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000800:00000200:3.0:1784948671.092254:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948671.092258:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009ff40a64. 00000400:00000200:3.0:1784948671.092262:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948671.092266:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784948671.092270:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784948671.092274:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948671.092388:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948671.092434:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948671.092439:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948671.092441:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948671.092443:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948671.092454:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948671.092468:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x7ecd 00000800:00000001:0.0:1784948671.092470:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948671.092473:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948671.092482:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948671.092485:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948671.092488:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948671.092493:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000400:00000010:3.0:1784948671.092494:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000400:00000200:3.0:1784948671.092496:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948671.092498:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000db05a3db. 00000400:00000010:3.0:1784948671.092500:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003bd24629. 00000800:00000001:3.0:1784948671.092504:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948671.092506:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948671.732216:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948671.732229:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948671.732234:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948671.924016:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948671.924029:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948671.924031:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948671.924036:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948671.924039:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948671.924042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948671.924044:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948672.116224:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948672.116237:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:0.0:1784948672.116245:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948672.116248:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948672.116251:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f43fd4d9 00000400:00000010:0.0:1784948672.116254:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f43fd4d9. 00000400:00000200:0.0:1784948672.116260:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948672.116265:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:0.0:1784948672.116270:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000075318cf7. 00000400:00000200:0.0:1784948672.116280:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:0.0:1784948672.116285:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:0.0:1784948672.116292:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000100:00000001:3.0:1784948672.947958:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948672.947969:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948672.947970:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948672.947975:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948672.947977:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948672.947980:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948672.947982:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948673.140125:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948673.140138:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:0.0:1784948673.140148:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948673.140154:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948673.140161:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000400:00000010:0.0:1784948673.140166:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000400:00000200:0.0:1784948673.140172:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948673.140183:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:0.0:1784948673.140190:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032fe57fd. 00000400:00000200:0.0:1784948673.140197:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:0.0:1784948673.140205:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:0.0:1784948673.140212:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:0.0:1784948673.140218:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:0.0:1784948673.140221:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948673.140223:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948673.140225:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:0.0:1784948673.140227:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000400:00000200:0.0:1784948673.140230:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948673.140233:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:0.0:1784948673.140235:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000070ada26c. 00000400:00000200:0.0:1784948673.140238:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:0.0:1784948673.140240:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:0.0:1784948673.140242:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000800:00000200:3.0:1784948673.140321:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948673.140326:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000820f02b3. 00000400:00000200:3.0:1784948673.140332:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948673.140338:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784948673.140345:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784948673.140347:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948673.140473:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948673.140502:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948673.140508:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948673.140510:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948673.140513:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948673.140524:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948673.140539:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x7ed1 00000800:00000001:3.0:1784948673.140551:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948673.140558:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948673.140560:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948673.140565:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948673.140570:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000400:00000010:3.0:1784948673.140572:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000400:00000200:3.0:1784948673.140575:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948673.140577:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000007367a1f2. 00000400:00000010:3.0:1784948673.140580:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003bd24629. 00000800:00000001:3.0:1784948673.140585:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948673.140587:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948673.780158:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948673.780166:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948673.780172:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948673.972008:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948673.972024:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948673.972028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948673.972035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948673.972040:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948673.972044:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948673.972048:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948674.164224:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948674.164239:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:0.0:1784948674.164248:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.164251:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.164254:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:0.0:1784948674.164257:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000400:00000200:0.0:1784948674.164263:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.164268:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:0.0:1784948674.164273:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000cf77ef26. 00000400:00000200:0.0:1784948674.164279:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:0.0:1784948674.164283:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:0.0:1784948674.164288:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:0.0:1784948674.164292:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.24@tcp(00000000d587ab1c) state 0x840 00000400:00000001:0.0:1784948674.164295:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.164296:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.164298:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000400:00000010:0.0:1784948674.164300:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000400:00000200:0.0:1784948674.164302:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.164305:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.24@tcp: -114 00000400:00000010:0.0:1784948674.164307:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004215ddb7. 00000400:00000200:0.0:1784948674.164311:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.24@tcp:-110 00000400:00000200:0.0:1784948674.164314:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.24@tcp(00000000d587ab1c) state 0x2040 rc -110 00000400:00000200:0.0:1784948674.164317:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.24@tcp 00000400:00000200:0.0:1784948674.164725:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948674.164750:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000010:0.0:1784948674.164770:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000070ada26c. 00000400:00000010:0.0:1784948674.164785:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00000400:00000010:0.0:1784948674.164806:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000771e09fe. 00000400:00000200:0.0:1784948674.164826:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:0.0:1784948674.164859:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:0.0:1784948674.164867:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.164872:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.164876:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.164880:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:0.0:1784948674.164884:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3082:3082:252:4294967295] 111.222.142.1@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:0.0:1784948674.164892:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:0.0:1784948674.164901:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.164941:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000800:00000010:0.0:1784948674.164949:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006069fb41. 00000800:00000200:0.0:1784948674.164965:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:0.1:1784948674.164987:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:0.0:1784948674.165003:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:0.0:1784948674.165015:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:0.0:1784948674.165036:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000010:0.0:1784948674.165051:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:0.0:1784948674.165064:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000400:00000010:0.0:1784948674.165076:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002026f8a1. 00000400:00000200:0.0:1784948674.165092:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:0.0:1784948674.165123:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:0.0:1784948674.165133:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.165139:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.165143:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.165148:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:0.0:1784948674.165153:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3083:3083:251:4294967295] 111.222.173.2@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:0.0:1784948674.165163:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:0.0:1784948674.165174:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.165185:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:0.0:1784948674.165196:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005d1384a4. 00000800:00000200:0.0:1784948674.165222:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784948674.165241:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:0.0:1784948674.165280:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:0.0:1784948674.165298:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:0.0:1784948674.165324:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000010:0.0:1784948674.165343:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000015713dee. 00000400:00000010:0.0:1784948674.165353:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000400:00000010:0.0:1784948674.165373:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007f814b08. 00000400:00000200:0.0:1784948674.165395:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:0.0:1784948674.165417:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:0.0:1784948674.165433:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.165440:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.165444:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.165449:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:0.0:1784948674.165454:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3084:3084:250:4294967295] 111.222.142.2@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:0.0:1784948674.165461:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:0.0:1784948674.165471:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.165481:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000800:00000010:0.0:1784948674.165512:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006ee15577. 00000800:00000200:0.0:1784948674.165537:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.0:1784948674.165551:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784948674.165579:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:0.1:1784948674.165593:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1784948674.165617:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.2@tcp 00000800:00000010:0.0:1784948674.165625:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006ee15577. 00000400:00000200:0.0:1784948674.165637:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784948674.165680:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784948674.165691:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948674.165701:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:0.0:1784948674.165813:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:0.0:1784948674.165844:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:0.0:1784948674.165858:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 rc 0 00000400:00000200:0.0:1784948674.165878:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000010:0.0:1784948674.165895:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000032fe57fd. 00000400:00000010:0.0:1784948674.165940:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000089c8262f. 00000400:00000010:0.0:1784948674.165959:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001542e92a. 00000400:00000200:0.0:1784948674.165982:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:0.0:1784948674.166010:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:0.0:1784948674.166023:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.166029:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.166033:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.166035:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:0.0:1784948674.166039:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3085:3085:250:4294967295] 111.222.173.3@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:0.0:1784948674.166044:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:0.0:1784948674.166050:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.166058:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:0.0:1784948674.166066:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006ee15577. 00000800:00000200:0.0:1784948674.166086:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:0.1:1784948674.166110:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:0.0:1784948674.166134:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:0.0:1784948674.166152:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:0.0:1784948674.166177:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000010:0.0:1784948674.166196:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000075318cf7. 00000400:00000010:0.0:1784948674.166211:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f43fd4d9. 00000400:00000010:0.0:1784948674.166227:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007864ed56. 00000400:00000200:0.0:1784948674.166249:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:0.0:1784948674.166281:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:0.0:1784948674.166295:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.166301:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.166304:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.166307:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:0.0:1784948674.166310:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3086:3086:249:4294967295] 111.222.173.4@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:0.0:1784948674.166316:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:0.0:1784948674.166322:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.166329:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000800:00000010:0.0:1784948674.166333:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005e0a2c1e. 00000800:00000200:0.0:1784948674.166349:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784948674.166367:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:0.0:1784948674.166400:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:0.0:1784948674.166411:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:0.0:1784948674.166436:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000010:0.0:1784948674.166456:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000010:0.0:1784948674.166465:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000400:00000010:0.0:1784948674.166476:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000541952f4. 00000400:00000200:0.0:1784948674.166490:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:0.0:1784948674.166527:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:0.0:1784948674.166536:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.166546:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.166551:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.166554:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:0.0:1784948674.166558:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3087:3087:248:4294967295] 111.222.142.4@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:0.0:1784948674.166563:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:0.0:1784948674.166569:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.166576:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:0.0:1784948674.166584:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002429e93e. 00000800:00000200:0.0:1784948674.166597:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.0:1784948674.166606:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784948674.166628:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784948674.166634:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1784948674.166652:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.4@tcp 00000800:00000010:0.0:1784948674.166685:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002429e93e. 00000400:00000200:0.0:1784948674.166696:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784948674.166715:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784948674.166732:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948674.166739:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:0.0:1784948674.166744:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:0.0:1784948674.166752:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:0.0:1784948674.166754:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 rc 0 00000400:00000200:0.0:1784948674.166758:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:0.0:1784948674.166763:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b8810b40. 00000400:00000010:0.0:1784948674.166766:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000400:00000010:0.0:1784948674.166770:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007f70671d. 00000400:00000200:0.0:1784948674.166773:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:0.0:1784948674.166779:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:0.0:1784948674.166783:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.166786:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.166789:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.166792:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:0.0:1784948674.166795:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3088:3088:248:4294967295] 111.222.142.5@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:0.0:1784948674.166801:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:0.0:1784948674.166808:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.166814:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:0.0:1784948674.166819:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002429e93e. 00000800:00000200:0.0:1784948674.166837:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784948674.166852:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:0.0:1784948674.166861:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:0.0:1784948674.166866:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:0.0:1784948674.166872:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:0.0:1784948674.166885:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e120de1c. 00000400:00000010:0.0:1784948674.166895:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000400:00000010:0.0:1784948674.166942:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009c70d9ae. 00000400:00000200:0.0:1784948674.166957:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:0.0:1784948674.166985:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:0.0:1784948674.166989:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.166992:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.166995:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.166998:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:0.0:1784948674.167001:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3089:3089:247:4294967295] 111.222.173.6@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:0.0:1784948674.167006:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:0.0:1784948674.167012:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167018:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:0.0:1784948674.167024:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c6406cdb. 00000800:00000200:0.0:1784948674.167039:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784948674.167045:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:0.0:1784948674.167072:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:0.0:1784948674.167078:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:0.0:1784948674.167091:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:0.0:1784948674.167102:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004215ddb7. 00000400:00000010:0.0:1784948674.167108:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000400:00000010:0.0:1784948674.167118:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000029a19359. 00000400:00000200:0.0:1784948674.167125:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:0.0:1784948674.167135:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:0.0:1784948674.167141:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.167145:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.167149:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.167153:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:0.0:1784948674.167159:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3090:3090:246:4294967295] 111.222.142.6@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:0.0:1784948674.167167:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:0.0:1784948674.167177:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167185:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:0.0:1784948674.167272:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000aae658b7. 00000800:00000200:0.0:1784948674.167282:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.0:1784948674.167287:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784948674.167295:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784948674.167299:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1784948674.167306:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.6@tcp 00000800:00000010:0.0:1784948674.167308:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000aae658b7. 00000400:00000200:0.0:1784948674.167312:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784948674.167317:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784948674.167322:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948674.167325:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:0.0:1784948674.167328:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:0.0:1784948674.167336:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:0.0:1784948674.167338:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 rc 0 00000400:00000200:0.0:1784948674.167342:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:0.0:1784948674.167347:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005152bcfe. 00000400:00000010:0.0:1784948674.167351:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 00000400:00000010:0.0:1784948674.167354:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003ad675ec. 00000400:00000200:0.0:1784948674.167358:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:0.0:1784948674.167364:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:0.0:1784948674.167367:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.167370:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.167373:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.167376:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:0.0:1784948674.167380:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3091:3091:246:4294967295] 111.222.173.7@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:0.0:1784948674.167384:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:0.0:1784948674.167390:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167395:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:0.0:1784948674.167398:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000aae658b7. 00000800:00000200:0.0:1784948674.167403:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:0.1:1784948674.167407:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:0.0:1784948674.167410:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:0.0:1784948674.167412:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000400:00000200:0.0:1784948674.167415:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:0.0:1784948674.167421:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e7299d92. 00000400:00000010:0.0:1784948674.167423:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000553902b5. 00000400:00000010:0.0:1784948674.167426:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000dbce1d1e. 00000400:00000200:0.0:1784948674.167430:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:0.0:1784948674.167434:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:0.0:1784948674.167436:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.167439:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.167441:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.167444:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:0.0:1784948674.167447:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3092:3092:245:4294967295] 111.222.142.7@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:0.0:1784948674.167451:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:0.0:1784948674.167457:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167461:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:0.0:1784948674.167464:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e471e641. 00000800:00000200:0.0:1784948674.167468:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784948674.167471:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:0.0:1784948674.167475:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:0.0:1784948674.167477:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:0.0:1784948674.167479:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:0.0:1784948674.167483:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef171f31. 00000400:00000010:0.0:1784948674.167486:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002e7a7cb2. 00000400:00000010:0.0:1784948674.167489:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008fec2664. 00000400:00000200:0.0:1784948674.167492:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:0.0:1784948674.167495:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:0.0:1784948674.167498:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.167500:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.167504:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.167506:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:0.0:1784948674.167509:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3093:3093:244:4294967295] 111.222.173.8@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:0.0:1784948674.167513:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:0.0:1784948674.167519:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167523:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:0.0:1784948674.167551:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001adbd2d9. 00000800:00000200:0.0:1784948674.167556:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784948674.167561:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:0.0:1784948674.167579:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:0.0:1784948674.167582:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:0.0:1784948674.167586:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:0.0:1784948674.167591:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000098dd29a0. 00000400:00000010:0.0:1784948674.167595:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000400:00000010:0.0:1784948674.167599:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007834bcf1. 00000400:00000200:0.0:1784948674.167602:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:0.0:1784948674.167607:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:0.0:1784948674.167611:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.167614:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.167617:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.167621:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:0.0:1784948674.167625:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3094:3094:243:4294967295] 111.222.142.8@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:0.0:1784948674.167632:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:0.0:1784948674.167638:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167644:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:0.0:1784948674.167648:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000faf98f5b. 00000800:00000200:0.0:1784948674.167652:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.0:1784948674.167682:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784948674.167697:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:0.1:1784948674.167700:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1784948674.167705:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.8@tcp 00000800:00000010:0.0:1784948674.167707:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000faf98f5b. 00000400:00000200:0.0:1784948674.167711:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784948674.167718:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784948674.167723:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948674.167726:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:0.0:1784948674.167731:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:0.0:1784948674.167738:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:0.0:1784948674.167742:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 rc 0 00000400:00000200:0.0:1784948674.167746:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:0.0:1784948674.167758:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000244a9cae. 00000400:00000010:0.0:1784948674.167762:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000236111b7. 00000400:00000010:0.0:1784948674.167769:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a9029990. 00000400:00000200:0.0:1784948674.167774:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:0.0:1784948674.167780:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:0.0:1784948674.167784:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.167787:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.167790:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.167793:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:0.0:1784948674.167798:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3095:3095:243:4294967295] 111.222.142.9@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:0.0:1784948674.167805:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:0.0:1784948674.167811:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167817:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:0.0:1784948674.167822:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000faf98f5b. 00000800:00000200:0.0:1784948674.167826:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784948674.167831:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:0.0:1784948674.167835:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:0.0:1784948674.167837:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:0.0:1784948674.167841:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000001:0.0:1784948674.167843:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.167845:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.167847:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:0.0:1784948674.167849:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000400:00000200:0.0:1784948674.167852:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.167855:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:0.0:1784948674.167858:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000015713dee. 00000400:00000200:0.0:1784948674.167864:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-5 00000400:00000200:0.0:1784948674.167867:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -5 00000400:00000200:0.0:1784948674.167870:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:0.0:1784948674.167874:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:0.0:1784948674.167877:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000015713dee. 00000400:00000010:0.0:1784948674.167880:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000400:00000010:0.0:1784948674.167883:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bfbf2878. 00000400:00000200:0.0:1784948674.167886:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:0.0:1784948674.167891:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:0.0:1784948674.167894:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.167897:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.167899:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.167902:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:0.0:1784948674.167935:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3096:3096:242:4294967295] 111.222.173.10@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:0.0:1784948674.167940:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:0.0:1784948674.167946:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.167953:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:0.0:1784948674.167957:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b0d293c7. 00000800:00000200:0.0:1784948674.167965:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784948674.167971:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:0.0:1784948674.168014:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:0.0:1784948674.168021:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:0.0:1784948674.168030:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:0.0:1784948674.168039:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002fbb6924. 00000400:00000010:0.0:1784948674.168045:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000400:00000010:0.0:1784948674.168053:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000073e0dd6a. 00000400:00000200:0.0:1784948674.168058:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:0.0:1784948674.168069:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:0.0:1784948674.168075:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168079:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168084:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168089:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:0.0:1784948674.168094:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3097:3097:241:4294967295] 111.222.142.10@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:0.0:1784948674.168102:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:0.0:1784948674.168112:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168121:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:0.0:1784948674.168126:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076d8913a. 00000800:00000200:0.0:1784948674.168134:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.0:1784948674.168138:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784948674.168147:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784948674.168152:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1784948674.168159:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.10@tcp 00000800:00000010:0.0:1784948674.168163:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000076d8913a. 00000400:00000200:0.0:1784948674.168168:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784948674.168174:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784948674.168180:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948674.168184:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:0.0:1784948674.168190:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:0.0:1784948674.168199:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:0.0:1784948674.168203:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 rc 0 00000400:00000200:0.0:1784948674.168209:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:0.0:1784948674.168216:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f11303f8. 00000400:00000010:0.0:1784948674.168221:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000400:00000010:0.0:1784948674.168228:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006fba1cc1. 00000400:00000200:0.0:1784948674.168234:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:0.0:1784948674.168241:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:0.0:1784948674.168245:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168248:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168251:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168254:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:0.0:1784948674.168257:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3098:3098:241:4294967295] 111.222.173.11@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:0.0:1784948674.168262:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:0.0:1784948674.168269:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168274:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:0.0:1784948674.168277:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076d8913a. 00000800:00000200:0.0:1784948674.168282:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784948674.168286:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:0.0:1784948674.168290:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:0.0:1784948674.168293:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:0.0:1784948674.168298:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:0.0:1784948674.168303:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000013f68999. 00000400:00000010:0.0:1784948674.168305:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000400:00000010:0.0:1784948674.168309:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d6af1288. 00000400:00000200:0.0:1784948674.168313:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:0.0:1784948674.168317:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:0.0:1784948674.168320:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168323:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168325:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168328:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:0.0:1784948674.168331:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3099:3099:240:4294967295] 111.222.142.11@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:0.0:1784948674.168335:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:0.0:1784948674.168343:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168348:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:0.0:1784948674.168351:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c70e9e66. 00000800:00000200:0.0:1784948674.168355:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784948674.168359:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:0.0:1784948674.168362:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:0.0:1784948674.168364:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:0.0:1784948674.168367:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:0.0:1784948674.168371:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005cc3cd5d. 00000400:00000010:0.0:1784948674.168373:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000400:00000010:0.0:1784948674.168376:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000073805bac. 00000400:00000200:0.0:1784948674.168379:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:0.0:1784948674.168383:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:0.0:1784948674.168386:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168388:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168390:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168392:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:0.0:1784948674.168395:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3100:3100:239:4294967295] 111.222.173.12@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:0.0:1784948674.168399:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:0.0:1784948674.168405:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168409:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:0.0:1784948674.168412:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000868cf222. 00000800:00000200:0.0:1784948674.168415:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:0.1:1784948674.168419:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:0.0:1784948674.168438:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:0.0:1784948674.168441:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:0.0:1784948674.168445:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:0.0:1784948674.168449:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ead2c724. 00000400:00000010:0.0:1784948674.168453:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 00000400:00000010:0.0:1784948674.168458:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000c1ba2e6. 00000400:00000200:0.0:1784948674.168465:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:0.0:1784948674.168472:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:0.0:1784948674.168476:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168479:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168481:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168483:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:0.0:1784948674.168487:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3101:3101:238:4294967295] 111.222.142.12@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:0.0:1784948674.168493:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:0.0:1784948674.168500:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168505:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:0.0:1784948674.168510:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000595fa8f8. 00000800:00000200:0.0:1784948674.168514:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:0.0:1784948674.168517:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784948674.168521:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:0.1:1784948674.168524:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1784948674.168528:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.12@tcp 00000800:00000010:0.0:1784948674.168530:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000595fa8f8. 00000400:00000200:0.0:1784948674.168533:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784948674.168538:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784948674.168542:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948674.168545:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:0.0:1784948674.168549:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:0.0:1784948674.168557:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:0.0:1784948674.168559:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 rc 0 00000400:00000200:0.0:1784948674.168563:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:0.0:1784948674.168568:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000859865aa. 00000400:00000010:0.0:1784948674.168572:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000400:00000010:0.0:1784948674.168576:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bb67e12b. 00000400:00000200:0.0:1784948674.168582:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:0.0:1784948674.168587:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:0.0:1784948674.168590:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168593:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168596:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168598:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:0.0:1784948674.168601:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3102:3102:238:4294967295] 111.222.173.13@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:0.0:1784948674.168606:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:0.0:1784948674.168611:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168616:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:0.0:1784948674.168620:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000595fa8f8. 00000800:00000200:0.0:1784948674.168625:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784948674.168628:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:0.0:1784948674.168633:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:0.0:1784948674.168635:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:0.0:1784948674.168638:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:0.0:1784948674.168646:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001df3b40e. 00000400:00000010:0.0:1784948674.168649:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000400:00000010:0.0:1784948674.168653:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008c12d7f6. 00000400:00000200:0.0:1784948674.168678:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:0.0:1784948674.168683:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:0.0:1784948674.168686:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168689:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168691:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168694:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:0.0:1784948674.168697:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3103:3103:237:4294967295] 111.222.142.13@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:0.0:1784948674.168702:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:0.0:1784948674.168709:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168713:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:0.0:1784948674.168719:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000eafb2cdc. 00000800:00000200:0.0:1784948674.168723:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:0.1:1784948674.168730:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:0.0:1784948674.168734:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:0.0:1784948674.168738:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:0.0:1784948674.168744:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:0.0:1784948674.168752:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003cdea007. 00000400:00000010:0.0:1784948674.168758:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000400:00000010:0.0:1784948674.168762:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000341744ce. 00000400:00000200:0.0:1784948674.168766:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:0.0:1784948674.168772:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:0.0:1784948674.168777:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168780:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168783:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168786:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:0.0:1784948674.168789:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3104:3104:236:4294967295] 111.222.173.14@tcp LPNI seq info [20:20:8:4294967295] 00000400:00000200:0.0:1784948674.168794:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:0.0:1784948674.168800:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168809:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:0.0:1784948674.168816:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006487a147. 00000800:00000200:0.0:1784948674.168825:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:0.1:1784948674.168833:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:0.0:1784948674.168841:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:0.0:1784948674.168845:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:0.0:1784948674.168851:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:0.0:1784948674.168859:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000069d2b91d. 00000400:00000010:0.0:1784948674.168863:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000400:00000010:0.0:1784948674.168869:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002cc5a29e. 00000400:00000200:0.0:1784948674.168874:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:0.0:1784948674.168884:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:0.0:1784948674.168889:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.168894:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.168898:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.168952:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:0.0:1784948674.168962:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3105:3105:235:4294967295] 111.222.142.14@tcp LPNI seq info [20:20:8:4294967295] 00000400:00000200:0.0:1784948674.168973:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:0.0:1784948674.168987:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.168998:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:0.0:1784948674.169005:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f8a82730. 00000800:00000200:0.0:1784948674.169014:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.0:1784948674.169019:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 10 secs later) 00000800:00000200:0.1:1784948674.169027:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.1:1784948674.169031:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 10 secs later) 00000800:00000100:0.0:1784948674.169037:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.14@tcp 00000800:00000010:0.0:1784948674.169041:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f8a82730. 00000400:00000200:0.0:1784948674.169047:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784948674.169053:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784948674.169058:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948674.169062:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:0.0:1784948674.169068:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:0.0:1784948674.169077:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:0.0:1784948674.169081:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 rc 0 00000400:00000200:0.0:1784948674.169087:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:0.0:1784948674.169094:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eff166f6. 00000400:00000010:0.0:1784948674.169099:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00000400:00000010:0.0:1784948674.169106:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004429599f. 00000400:00000200:0.0:1784948674.169112:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:0.0:1784948674.169121:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:0.0:1784948674.169126:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.169130:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.169135:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.169139:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:0.0:1784948674.169144:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3106:3106:235:4294967295] 111.222.173.15@tcp LPNI seq info [19:19:8:4294967295] 00000400:00000200:0.0:1784948674.169152:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:0.0:1784948674.169162:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.169171:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:0.0:1784948674.169177:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f8a82730. 00000800:00000200:0.0:1784948674.169184:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:0.1:1784948674.169191:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:0.0:1784948674.169196:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:0.0:1784948674.169199:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:0.0:1784948674.169203:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:0.0:1784948674.169207:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000194fc0c0. 00000400:00000010:0.0:1784948674.169210:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000400:00000010:0.0:1784948674.169215:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000075884760. 00000400:00000200:0.0:1784948674.169219:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:0.0:1784948674.169224:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:0.0:1784948674.169228:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948674.169231:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.169233:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948674.169236:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:0.0:1784948674.169239:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3107:3107:234:4294967295] 111.222.142.15@tcp LPNI seq info [18:18:8:4294967295] 00000400:00000200:0.0:1784948674.169244:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:0.0:1784948674.169250:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.169254:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:0.0:1784948674.169257:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008746b1e6. 00000800:00000200:0.0:1784948674.169261:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:0.1:1784948674.169265:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:0.0:1784948674.169268:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:0.0:1784948674.169271:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:0.0:1784948674.169274:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:0.0:1784948674.169277:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.169279:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.169280:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:0.0:1784948674.169282:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000400:00000200:0.0:1784948674.169286:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.169289:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:0.0:1784948674.169292:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089e37819. 00000400:00000200:0.0:1784948674.169297:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-5 00000400:00000200:0.0:1784948674.169299:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -5 00000400:00000200:0.0:1784948674.169303:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:0.0:1784948674.169306:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.25@tcp(000000003bd24629) state 0x40 00000400:00000010:0.0:1784948674.169309:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000010:0.0:1784948674.169313:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000400:00000010:0.0:1784948674.169316:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000058ef8883. 00000400:00000200:0.0:1784948674.169321:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.25@tcp 00000400:00000200:0.0:1784948674.169326:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.173.25@tcp local destination 00000400:00000200:0.0:1784948674.169330:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948674.169333:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.173.25@tcp 00000400:00000200:0.0:1784948674.169338:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.173.25@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:0.0:1784948674.169342:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3108:3108:233:4294967295] 111.222.173.25@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:0.0:1784948674.169346:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.25@tcp 00000400:00000200:0.0:1784948674.169352:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.25@tcp(111.222.173.25@tcp:111.222.173.25@tcp) : GET try# 0 00000800:00000200:0.0:1784948674.169356:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.25@tcp 00000800:00000010:0.0:1784948674.169359:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002aeee8e9. 00000800:00000010:0.0:1784948674.169366:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 0000000075e18d76. 00000800:00000010:0.0:1784948674.169369:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 0000000005acc249. 00000800:00000200:0.0:1784948674.169374:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000075e18d76] -> 12345-111.222.173.25@tcp (2) 00000800:00000200:0.1:1784948674.169380:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000075e18d76] -> 12345-111.222.173.25@tcp (2) 00000400:00000200:0.0:1784948674.169398:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.25@tcp 00000400:00000200:0.0:1784948674.169402:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.25@tcp(000000003bd24629) state 0x240 rc 0 00000400:00000200:0.0:1784948674.169406:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 00000400:00000001:0.0:1784948674.169409:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.169410:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.169413:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000400:00000010:0.0:1784948674.169415:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000400:00000200:0.0:1784948674.169418:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.169423:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:0.0:1784948674.169426:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004215ddb7. 00000400:00000200:0.0:1784948674.169431:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-5 00000400:00000200:0.0:1784948674.169434:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -5 00000400:00000200:0.0:1784948674.169438:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:0.0:1784948674.169443:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:0.0:1784948674.169446:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.169448:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.169449:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:0.0:1784948674.169451:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000400:00000200:0.0:1784948674.169454:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.169458:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:0.0:1784948674.169461:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000098dd29a0. 00000400:00000200:0.0:1784948674.169466:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-5 00000400:00000200:0.0:1784948674.169471:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -5 00000400:00000200:0.0:1784948674.169477:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000400:00000200:0.0:1784948674.169482:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:0.0:1784948674.169484:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.169485:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.169487:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:0.0:1784948674.169489:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000400:00000200:0.0:1784948674.169492:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.169494:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:0.0:1784948674.169496:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002fbb6924. 00000400:00000200:0.0:1784948674.169500:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-5 00000400:00000200:0.0:1784948674.169503:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -5 00000400:00000200:0.0:1784948674.169506:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000400:00000200:0.0:1784948674.169510:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:0.0:1784948674.169513:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.169514:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.169515:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000400:00000010:0.0:1784948674.169517:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000400:00000200:0.0:1784948674.169519:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.169521:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:0.0:1784948674.169523:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ead2c724. 00000400:00000200:0.0:1784948674.169528:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-5 00000400:00000200:0.0:1784948674.169530:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -5 00000400:00000200:0.0:1784948674.169533:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000400:00000200:0.0:1784948674.169535:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:0.0:1784948674.169537:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948674.169538:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948674.169540:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:0.0:1784948674.169541:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000400:00000200:0.0:1784948674.169544:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948674.169547:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:0.0:1784948674.169549:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000069d2b91d. 00000400:00000200:0.0:1784948674.169554:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-5 00000400:00000200:0.0:1784948674.169556:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -5 00000400:00000200:0.0:1784948674.169559:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00000100:00000001:3.0:1784948674.996081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948674.996097:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948674.996105:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948674.996114:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948674.996121:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948674.996128:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.996137:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.996377:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948674.996385:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948674.996391:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948674.996403:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948674.996408:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948674.996419:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.996435:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948674.996441:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948674.996457:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948674.996466:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948674.996480:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948674.996496:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289644928:0@lo:400: 00000100:00000001:3.0:1784948674.996508:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948674.996518:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948674.996526:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948674.996555:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948674.996571:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005e186afc. 02000000:00000001:3.0:1784948674.996583:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948674.996601:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c6713e8b. 00000400:00000010:3.0:1784948674.996621:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000200:3.0:1784948674.996638:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289644928, portal 25 00000100:00000001:3.0:1784948674.996673:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948674.996679:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451651456 : -115544257900160 : ffff96e9c18d5180) 00000100:00000040:3.0:1784948674.996699:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948674.996733:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948674.996748:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948674.996760:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000100:00000200:3.0:1784948674.996773:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289644928, offset 0 00000400:00000200:3.0:1784948674.996797:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948674.996823:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948674.996852:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6433f80 00000400:00000200:3.0:1784948674.996871:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x752d [2] + 0 00000400:00000010:3.0:1784948674.996887:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004aad1993. 00000400:00000200:3.0:1784948674.996903:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000059548475 00000400:00000200:3.0:1784948674.996950:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948674.996961:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948674.996987:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000059548475 00000400:00000010:3.0:1784948674.996995:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000059548475. 00000100:00000001:3.0:1784948674.997005:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948674.997010:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784948674.997019:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000093c39287 x1871651289644928 msgsize 224 00000100:00100000:3.0:1784948674.997040:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784948674.997048:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784948674.997071:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948674.997086:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000400:00000010:3.0:1784948674.997094:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000100:00000001:3.0:1784948674.997103:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784948674.997112:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948674.997132:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948674.997140:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948674.997154:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.997159:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948674.997168:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.997177:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305614379 00000100:00000001:3.0:1784948674.997191:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305614340 00000100:00000001:3.0:1784948674.997202:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305614379 00000100:00000001:3.0:1784948674.997213:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305614340 00000100:00000001:3.0:1784948674.997222:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305614379 00000100:00000001:3.0:1784948674.997246:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305614340 00000100:00000001:3.0:1784948674.997262:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305614379 00000100:00000001:3.0:1784948674.997277:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305614340 00000100:00000001:3.0:1784948674.997290:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948674.997298:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.997307:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.997314:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948674.997325:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948674.997332:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948674.997339:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948674.997344:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948674.997349:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.997354:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948674.997409:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948674.997431:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948674.997439:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948674.997461:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948674.997490:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948674.997518:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434000 00000400:00000200:3.0:1784948674.997546:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 147536 00000800:00000001:3.0:1784948674.997567:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948674.997580:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948674.997590:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948674.997608:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948674.997634:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948674.997641:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:3.0:1784948674.997677:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000002d77f222. 00000100:00000040:3.0:1784948674.997686:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000002d77f222 x1871651289645056 msgsize 224 00000100:00100000:3.0:1784948674.997705:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784948674.997730:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:3.0:1784948674.997745:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948674.997752:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948674.998202:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000d6df185. 00000100:00100000:1.0:1784948674.998212:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000002d77f222 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289645056:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948674.998221:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289645056 00000020:00000001:1.0:1784948674.998227:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948674.998234:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948674.998241:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948674.998250:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948674.998254:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948674.998259:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948674.998266:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948674.998274:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948674.998279:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948674.998284:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948674.998291:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948674.998297:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948674.998302:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948674.998309:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001076c096. 02000000:00000001:1.0:1784948674.998317:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948674.998324:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948674.998333:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948674.998341:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289645056 00010000:00000001:1.0:1784948674.998348:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948674.998357:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000002d77f222 x1871651289645056/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:335/0 lens 224/224 e 0 to 0 dl 1784948685 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948674.998368:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948674.998371:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948674.998374:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=141 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948674.998382:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948674.998387:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948674.998394:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948674.998402:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948674.998407:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948674.998414:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948674.998425:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948674.998427:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948674.998433:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:1.0:1784948674.998436:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000001:3.0:1784948674.998445:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948674.998451:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948674.998457:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c6713e8b. 00000400:00000200:3.0:1784948674.998469:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:3.0:1784948674.998476:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:3.0:1784948674.998485:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948674.998487:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289645056, offset 224 00000100:00000200:3.0:1784948674.998497:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948674.998511:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000200:3.0:1784948674.998535:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948674.998538:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948674.998562:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3110:3110:232:4294967295] 192.168.202.137@tcp LPNI seq info [2018:2018:8:4294967295] 00000100:00000001:3.0:1784948674.998565:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948674.998571:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948674.998577:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948674.998586:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948674.998592:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784948674.998598:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948674.998599:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948674.998610:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948674.998619:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=146 v=5 (5 5 5 5) 00000800:00000200:1.0:1784948674.998620:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00001000:3.0:1784948674.998634:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=146 v=5 (1 1 1 1) 00000800:00000010:1.0:1784948674.998635:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000062b4a223. 00000800:00000200:1.0:1784948674.998672:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948674.998680:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000800:00000200:1.0:1784948674.998689:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784948674.998689:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948674.998701:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000062b4a223 type 1, nob 320 niov 1 nkiov 1 00010000:00000001:3.0:1784948674.998702:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784948674.998709:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.998716:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948674.998727:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948674.998728:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948674.998743:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948674.998753:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948674.998758:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948674.998760:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948674.998765:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784948674.998768:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948674.998773:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948674.998777:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002d77f222 x1871651289645056/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:335/0 lens 224/224 e 0 to 0 dl 1784948685 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948674.998783:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948674.998826:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000002d77f222 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289645056:12345-192.168.202.137@tcp:400: Request processed in 642us (1135us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948674.998829:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289644928:0@lo:400: 00000100:00000001:3.0:1784948674.998850:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948674.998856:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 654 00000100:00000001:3.0:1784948674.998857:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948674.998865:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000040:3.0:1784948674.998866:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b55e20f2 x1871651289644928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948690 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948674.998875:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948674.998882:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948674.998889:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000010:1.0:1784948674.998890:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000056a1a23f. 02000000:00000001:3.0:1784948674.998894:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:1.0:1784948674.998898:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 02000000:00000010:3.0:1784948674.998900:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000005e186afc. 00000020:00000010:1.0:1784948674.998937:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 02000000:00000001:3.0:1784948674.998948:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:1.0:1784948674.998949:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000020:00000001:3.0:1784948674.998955:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1784948674.998957:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784948674.998961:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784948674.998968:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948674.998972:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000057c7ab4. 02000000:00000001:3.0:1784948674.998980:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948674.998984:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948674.998988:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b55e20f2. 00000100:00000001:3.0:1784948674.998998:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948674.999002:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948674.999007:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948674.999017:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948674.999019:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948674.999026:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948674.999030:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948674.999032:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:0.0:1784948674.999034:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784948674.999041:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948674.999043:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948674.999046:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000062b4a223. 00000800:00000001:1.0:1784948674.999049:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948674.999062:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948674.999064:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948674.999085:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434000 00000400:00000200:0.0:1784948674.999090:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948674.999102:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x7f45 [1] + 224 00000800:00000001:1.0:1784948674.999110:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948674.999119:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000800:00000001:1.0:1784948674.999121:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948674.999130:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000400:00000200:1.0:1784948674.999131:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948674.999143:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948674.999146:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948674.999151:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948674.999154:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000200:1.0:1784948674.999161:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a8233e67 x1871651289645056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948690 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948674.999176:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a8233e67 x1871651289645056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948690 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948674.999198:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948674.999207:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948674.999210:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948675.040001:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305614379, at 4305614380 00000100:00000010:1.0:1784948675.040020:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000100:00000040:0.0:1784948675.040023:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305614379, at 4305614380 00000100:00000010:0.0:1784948675.040033:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00000100:00000040:3.0:1784948675.040037:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305614379, at 4305614380 00000100:00000010:3.0:1784948675.040051:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f59314ac. 00000400:00000200:0.0:1784948675.188296:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948675.188307:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2861 00000400:00000001:0.0:1784948675.188315:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948675.188318:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948675.188321:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784948675.188324:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000400:00000200:0.0:1784948675.188329:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948675.188334:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:0.0:1784948675.188339:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eb983bef. 00000400:00000200:0.0:1784948675.188345:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:0.0:1784948675.188349:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:0.0:1784948675.188353:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000020:00000001:3.0:1784948675.828183:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948675.828192:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948675.828198:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948676.019999:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948676.020015:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948676.020018:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948676.020025:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948676.020028:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948676.020032:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948676.020035:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948676.212049:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948676.212066:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000005e1bb19b. 00000400:00000010:1.0:1784948676.212080:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000010:1.0:1784948676.212088:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000400:00000010:1.0:1784948676.212098:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009e0071c7. 00000400:00000200:1.0:1784948676.212112:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948676.212134:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948676.212144:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948676.212162:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3111:3111:233:4294967295] 192.168.202.137@tcp LPNI seq info [2019:2019:8:4294967295] 00000400:00000200:1.0:1784948676.212186:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948676.212213:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948676.212221:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948676.212236:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d104522b. 00000800:00000200:1.0:1784948676.212260:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948676.212291:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948676.212298:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d104522b type 2, nob 96 niov 1 nkiov 0 00000400:00000010:1.0:1784948676.212332:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 0000000025674e46. 00000400:02000040:1.0:1784948676.212340:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000400:00000200:0.0:1784948676.212358:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948676.212370:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2861 00000400:00000001:0.0:1784948676.212379:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948676.212383:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948676.212386:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000400:00000010:0.0:1784948676.212389:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000400:00000200:0.0:1784948676.212398:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948676.212404:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:0.0:1784948676.212410:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000063f28b6f. 00000400:00000200:0.0:1784948676.212417:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-110 00000400:00000200:0.0:1784948676.212420:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -110 00000400:00000200:0.0:1784948676.212425:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000800:00000200:3.0:1784948676.212520:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948676.212529:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d104522b. 00000800:00000001:0.0:1784948676.212538:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948676.212539:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948676.212552:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:0.0:1784948676.212560:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948676.212560:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784948676.212570:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948676.212571:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784948676.212583:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948676.212596:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948676.212613:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948676.212624:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948676.212634:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948676.212685:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3112:3112:234:4294967295] 192.168.202.137@tcp LPNI seq info [2020:2020:8:4294967295] 00000400:00000200:0.0:1784948676.212702:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948676.212713:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948676.212719:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a3cf4688. 00000800:00000200:0.0:1784948676.212725:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948676.212732:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948676.212736:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a3cf4688 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:0.0:1784948676.212895:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948676.212939:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a3cf4688. 00000400:00000200:0.0:1784948676.212945:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948676.212977:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000010:1.0:1784948676.216363:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 0000000025674e46. 00000020:00000001:1.0:1784948676.852180:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948676.852190:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948676.852197:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948677.044045:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948677.044062:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948677.044064:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948677.044070:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948677.044073:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948677.044077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948677.044080:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948677.236440:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948677.236468:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2861 00000400:00000001:0.0:1784948677.236484:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948677.236489:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948677.236494:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:0.0:1784948677.236498:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000400:00000200:0.0:1784948677.236508:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948677.236528:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:0.0:1784948677.236537:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000be26f34f. 00000400:00000200:0.0:1784948677.236548:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-110 00000400:00000200:0.0:1784948677.236552:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -110 00000400:00000200:0.0:1784948677.236559:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00000400:00000200:0.0:1784948677.236567:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2861 00000400:00000001:0.0:1784948677.236572:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948677.236575:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948677.236577:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000400:00000010:0.0:1784948677.236580:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000400:00000200:0.0:1784948677.236584:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948677.236589:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:0.0:1784948677.236593:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000041892d66. 00000400:00000200:0.0:1784948677.236600:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:0.0:1784948677.236603:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000400:00000200:0.0:1784948677.236606:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000020:00000001:3.0:1784948677.876281:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948677.876286:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948677.876290:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948678.068009:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948678.068026:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948678.068029:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948678.068035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948678.068039:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948678.068043:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948678.068046:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948678.260046:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948678.260058:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000eb33d71d. 00000400:00000010:1.0:1784948678.260069:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000010:1.0:1784948678.260076:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000010:1.0:1784948678.260084:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000041ec6752. 00000400:00000200:1.0:1784948678.260089:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948678.260109:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948678.260116:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948678.260121:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3113:3113:236:4294967295] 192.168.202.137@tcp LPNI seq info [2021:2021:8:4294967295] 00000400:00000200:1.0:1784948678.260128:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948678.260136:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948678.260143:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948678.260148:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000635133da. 00000800:00000200:1.0:1784948678.260155:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948678.260163:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948678.260168:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000635133da type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948678.260188:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 10952 health :800 00000800:00000200:3.0:1784948678.260386:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948678.260394:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000635133da. 00000400:00000200:3.0:1784948678.260405:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948678.260425:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:0.0:1784948678.260431:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948678.260434:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784948678.260443:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784948678.260453:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948678.260460:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948678.260469:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948678.260482:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948678.260501:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948678.260513:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948678.260527:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948678.260535:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3114:3114:236:4294967295] 192.168.202.137@tcp LPNI seq info [2022:2022:8:4294967295] 00000400:00000200:0.0:1784948678.260549:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948678.260557:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948678.260572:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a3cf4688. 00000800:00000200:0.0:1784948678.260580:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948678.260589:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948678.260595:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a3cf4688 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:0.0:1784948678.260772:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948678.260778:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a3cf4688. 00000400:00000200:0.0:1784948678.260785:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948678.260794:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948678.260807:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948678.260811:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948678.260819:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948678.260823:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948678.260840:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948678.260847:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948678.260857:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948678.260874:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x7f61 00000800:00000001:0.0:1784948678.260882:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948678.260895:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948678.260898:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948678.260936:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948678.260950:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:0.0:1784948678.260954:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000400:00000200:0.0:1784948678.260960:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948678.260964:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000eb33d71d. 00000400:00000010:0.0:1784948678.260969:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000800:00000001:0.0:1784948678.260976:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948678.260982:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948678.900158:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948678.900164:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948678.900169:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948679.092043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948679.092061:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948679.092064:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948679.092072:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948679.092076:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948679.092081:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948679.092086:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948679.924256:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948679.924262:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948679.924266:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948680.116069:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1784948680.116070:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948680.116084:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948680.116086:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948680.116089:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948680.116095:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784948680.116097:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a07b5f33. 00000100:00000001:3.0:1784948680.116099:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948680.116104:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948680.116107:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948680.116109:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948680.116119:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948680.116125:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948680.116133:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948680.116144:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00000001:1.0:1784948680.116154:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948680.116166:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948680.116181:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948680.116184:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948680.116192:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a07b5f33] to pc [ptlrpcd_00_03+3] req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948680.116229:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948680.116234:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948680.116239:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948680.116245:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948680.116249:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948680.116256:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b93d109. 00000020:00000040:1.0:1784948680.116268:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948680.116269:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948680.116281:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948680.116283:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948680.116295:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948680.116299:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948680.116313:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000008f5595bf. 00000100:00000001:1.0:1784948680.116321:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948680.116343:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948680.116356:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948680.116359:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948680.116378:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005b93d109] to pc [ptlrpcd_00_00+0] req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948680.116427:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948680.116446:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10957) 00000100:00000001:3.0:1784948680.116462:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948680.116468:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948680.116474:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948680.116489:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948680.116505:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948680.116511:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948680.116519:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948680.116524:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948680.116534:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948680.116548:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289645312:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948680.116558:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948680.116564:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948680.116567:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948680.116575:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948680.116582:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005e186afc. 02000000:00000001:3.0:1784948680.116587:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948680.116593:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c6713e8b. 00000400:00000010:3.0:1784948680.116601:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000200:3.0:1784948680.116642:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289645312, portal 10 00000100:00000001:3.0:1784948680.116647:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948680.116649:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451647872 : -115544257903744 : ffff96e9c18d4380) 00000100:00000040:3.0:1784948680.116680:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948696 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948680.116695:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948680.116702:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948680.116707:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000100:00000200:3.0:1784948680.116716:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289645312, offset 0 00000400:00000200:3.0:1784948680.116750:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948680.116772:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948680.116786:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948680.116793:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948680.116801:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948680.116813:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948680.116823:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3115:3115:236:4294967295] 192.168.202.137@tcp LPNI seq info [2023:2023:8:4294967295] 00000400:00000200:3.0:1784948680.116847:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948680.116854:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948680.116862:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003264a2f8. 00000800:00000200:3.0:1784948680.116877:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948680.116896:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948680.116960:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003264a2f8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948680.116980:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948680.116987:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2008 usecs (3 jiffies) 00000100:00000001:3.0:1784948680.116994:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305619460 00000100:00000010:3.0:1784948680.117007:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:0.0:1784948680.117008:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948680.117042:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948680.117043:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948680.117067:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305619460 00000100:00000010:3.0:1784948680.117075:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000afdabce. 00000100:00000001:3.0:1784948680.117097:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948680.117106:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305619460 00000100:00000010:3.0:1784948680.117111:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:3.0:1784948680.117120:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948680.117125:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305619460 00000100:00000010:3.0:1784948680.117135:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:3.0:1784948680.117142:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948680.117147:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948680.117156:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948680.117161:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948680.117164:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948680.117173:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948680.117177:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948680.117183:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948680.117188:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948680.117193:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948680.117199:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948680.117365:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948680.117369:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003264a2f8. 00000400:00000200:3.0:1784948680.117376:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948680.117388:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948680.117393:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000400:00000010:3.0:1784948680.117397:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000100:00000001:3.0:1784948680.117404:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784948680.117412:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948696 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948680.117426:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948680.117437:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948696 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948680.117456:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948680.117464:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948680.117724:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948680.117734:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289645184 02000000:00000001:1.0:1784948680.117741:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948680.117746:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948680.117752:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948680.117764:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948680.117845:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289645184 00000020:00000001:1.0:1784948680.117856:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948680.117865:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784948680.117879:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948680.117896:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784948680.117961:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784948680.117966:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948680.117974:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948680.117985:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000030655a79. 00000100:00000001:1.0:1784948680.117992:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948680.117999:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000030655a79. 00000020:00000010:1.0:1784948680.118018:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bf0e6e18. 00000020:00000010:1.0:1784948680.118028:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000042a244b9. 00000020:00000010:1.0:1784948680.118042:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:1.0:1784948680.118064:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948680.118075:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948680.118081:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948680.118087:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948680.118105:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948680.118132:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948680.118168:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948680.118180:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948680.118189:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948680.118195:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948680.118205:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 679 00000100:00000040:1.0:1784948680.118224:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948680.118226:0:342757:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948680.118255:0:342757:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948680.118255:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466828880 : -115544242722736 : ffff96e9c274e850) 00000100:00000040:1.0:1784948680.118281:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d416d8c1 x1871651289645184/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:341/0 lens 224/0 e 0 to 0 dl 1784948691 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948680.118318:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948680.118329:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000030655a79. 00000100:00000001:1.0:1784948680.118349:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948680.118363:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000030655a79. 00000100:00100000:1.0:1784948680.118377:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d416d8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289645184:12345-0@lo:400: 00000100:00000200:1.0:1784948680.118386:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289645184 00000020:00000001:1.0:1784948680.118397:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948680.118404:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948680.118414:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948680.118431:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948680.118440:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948680.118460:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948680.118468:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948680.118479:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948680.118497:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948680.118505:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948680.118516:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948680.118526:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948680.118537:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948680.118552:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000007cdbbc4. 02000000:00000001:1.0:1784948680.118568:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948680.118578:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948680.118588:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948680.118627:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289645184 00010000:00000001:1.0:1784948680.118635:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948680.118645:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d416d8c1 x1871651289645184/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:341/0 lens 224/224 e 0 to 0 dl 1784948691 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948680.118696:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948680.118707:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948680.118728:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=152 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948680.118743:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948680.118753:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948680.118759:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948680.118768:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948680.118775:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948680.118795:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948680.118801:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948680.118807:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000100:00000200:1.0:1784948680.118822:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289645184, offset 224 00000400:00000200:1.0:1784948680.118845:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948680.118882:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784948680.118962:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434080 00000400:00000200:1.0:1784948680.118982:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7f65 [1] + 224 00000400:00000200:1.0:1784948680.119008:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948680.119021:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000200:0.0:1784948680.119025:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784948680.119031:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784948680.119037:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000263f8d59. 00000100:00000001:1.0:1784948680.119050:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1784948680.119062:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1784948680.119064:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948680.119072:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948680.119085:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948680.119097:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948680.119110:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 00000800:00000001:3.0:1784948680.119113:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948680.119122:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000100:00000001:0.0:1784948680.119136:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948680.119146:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948680.119154:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:1.0:1784948680.119174:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948680.119200:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434100 00000400:00000200:3.0:1784948680.119242:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x7f6d [1] + 224 00000800:00000001:3.0:1784948680.119253:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948680.119264:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948680.119271:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948680.119277:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948680.119280:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:1.0:1784948680.119290:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000400:00000200:3.0:1784948680.119294:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948680.119303:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948680.119311:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784948680.119311:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948680.119313:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948680.119328:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948680.119332:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948680.119333:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784948680.119344:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1784948680.119352:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:0.0:1784948680.119356:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948680.119363:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000093b2e4e6. 00010000:00000001:1.0:1784948680.119369:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948680.119383:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948680.119384:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000010:0.0:1784948680.119396:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000020:00000001:1.0:1784948680.119401:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948680.119414:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784948680.119417:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d416d8c1 x1871651289645184/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:341/0 lens 224/224 e 0 to 0 dl 1784948691 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948680.119427:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948680.119431:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005b93d109 x1871651289645312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948680.119500:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948680.119505:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d416d8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289645184:12345-0@lo:400: Request processed in 1172us (2737us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948680.119557:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948680.119564:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948680.119564:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 679 00000100:00000001:3.0:1784948680.119567:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948680.119578:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:1.0:1784948680.119579:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 02000000:00000001:0.0:1784948680.119584:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948680.119586:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000001:3.0:1784948680.119586:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948680.119590:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948680.119599:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948680.119601:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:0.0:1784948680.119623:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948680.119627:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948680.119628:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000042a244b9. 00000100:00001000:0.0:1784948680.119635:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=152 v=5 (5 5 5 5) 00000020:00000010:1.0:1784948680.119640:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000100:00001000:0.0:1784948680.119645:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=152 v=5 (1 1 1 1) 00000020:00000010:1.0:1784948680.119651:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bf0e6e18. 00000100:00000001:0.0:1784948680.119653:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948680.119660:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948680.119663:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000001:0.0:1784948680.119664:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948680.119670:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948680.119675:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784948680.119676:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948680.119688:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000007cdbbc4. 00000100:00000040:0.0:1784948680.119689:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948680.119704:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784948680.119714:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948680.119721:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1784948680.119724:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948680.119738:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:1.0:1784948680.119740:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e5b8264. 00000100:00000001:0.0:1784948680.119746:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948680.119763:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948680.119788:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000037ca0070. 00000100:00100000:0.0:1784948680.119800:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a07b5f33 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289645184:0@lo:400: 00000100:00000001:0.0:1784948680.119827:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948680.119833:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948680.119846:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a07b5f33 x1871651289645184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948680.119956:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948680.119961:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948680.119968:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000f1904ca. 02000000:00000001:0.0:1784948680.119983:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948680.119993:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948680.120003:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948680.120008:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948680.120012:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 02000000:00000001:0.0:1784948680.120027:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948680.120031:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948680.120034:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a07b5f33. 00000100:00000001:0.0:1784948680.120047:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948680.120051:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948680.120058:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948680.120068:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948680.120072:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948680.120079:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948680.120082:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948680.160013:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305619499, at 4305619500 00000100:00000040:1.0:1784948680.160030:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305619499, at 4305619500 00000100:00000010:0.0:1784948680.160034:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:1.0:1784948680.160043:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000afdabce. 00000100:00000040:3.0:1784948680.160051:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305619499, at 4305619500 00000100:00000010:3.0:1784948680.160073:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000020:00000001:1.0:1784948680.948225:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948680.948233:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948680.948237:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948681.140088:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948681.140106:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948681.140110:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948681.140117:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948681.140122:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948681.140127:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948681.140130:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948681.972197:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948681.972207:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948681.972213:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948682.163993:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948682.164007:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948682.164009:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948682.164014:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948682.164017:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948682.164020:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948682.164023:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948682.996076:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948682.996081:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948682.996085:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948683.188033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948683.188048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948683.188050:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948683.188055:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948683.188058:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948683.188061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948683.188063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948684.020188:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948684.020199:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948684.020206:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948684.212090:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948684.212109:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948684.212112:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948684.212119:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948684.212123:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948684.212129:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948684.212133:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948685.044245:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948685.044252:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948685.044257:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784948685.236114:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948685.236134:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948685.236147:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000c8b06f4. 00000020:00000040:0.0:1784948685.236160:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948685.236169:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948685.236177:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948685.236186:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948685.236201:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a3cf4688. 00000100:00000001:0.0:1784948685.236215:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948685.236241:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948685.236302:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948685.236305:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948685.236317:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c8b06f4] to pc [ptlrpcd_00_01+1] req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948685.236378:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948685.236384:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948685.236391:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948685.236399:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948685.236406:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948685.236414:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ee0ebd63. 00000100:00000001:3.0:1784948685.236416:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784948685.236421:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948685.236426:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948685.236433:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948685.236433:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784948685.236443:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1784948685.236449:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948685.236453:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948685.236470:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f64a2f6c. 00000100:00000001:3.0:1784948685.236473:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948685.236479:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948685.236481:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948685.236485:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948685.236493:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948685.236500:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948685.236501:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1784948685.236512:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000c8b06f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289645440:0@lo:400: 00000100:00000001:3.0:1784948685.236531:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948685.236540:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1784948685.236542:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784948685.236544:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948685.236548:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948685.236554:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784948685.236561:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0ebd63] to pc [ptlrpcd_00_02+2] req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784948685.236597:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005e186afc. 02000000:00000001:3.0:1784948685.236606:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948685.236621:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000400:00000010:3.0:1784948685.236640:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000100:00000200:3.0:1784948685.236653:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289645440, portal 25 00000100:00000001:3.0:1784948685.236659:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948685.236663:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558297600 : -115544151254016 : ffff96e9c7e89c00) 00000100:00000001:0.0:1784948685.236666:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948685.236681:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948685.236683:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10962) 00000100:00000001:1.0:1784948685.236698:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948685.236705:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948685.236709:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948685.236718:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948685.236727:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948685.236737:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000040:1.0:1784948685.236740:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948685.236749:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289645440, offset 0 00000100:00000001:0.0:1784948685.236753:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948685.236763:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948685.236783:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948685.236813:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948685.236834:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434180 00000400:00000200:3.0:1784948685.236849:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7571 [2] + 0 00000400:00000010:3.0:1784948685.236858:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001004a12d. 00000400:00000200:3.0:1784948685.236869:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000e4acfb45 00000400:00000200:3.0:1784948685.236882:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948685.236893:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948685.236960:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e4acfb45 00000400:00000010:3.0:1784948685.236966:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000e4acfb45. 00000100:00000001:3.0:1784948685.236984:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948685.236988:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:3.0:1784948685.236991:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784948685.236996:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948685.237003:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000ab8619ba x1871651289645440 msgsize 224 02000000:00000001:1.0:1784948685.237008:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948685.237016:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784948685.237026:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 02000000:00000001:1.0:1784948685.237029:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948685.237034:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00100000:1.0:1784948685.237044:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ee0ebd63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289645568:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948685.237056:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948685.237063:0:337120:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:1.0:1784948685.237064:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948685.237074:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948685.237083:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948685.237087:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 02000000:00000010:1.0:1784948685.237093:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000da755428. 00000400:00000010:3.0:1784948685.237097:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 02000000:00000001:1.0:1784948685.237103:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948685.237108:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:1.0:1784948685.237120:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dbb46726. 00000100:00000200:3.0:1784948685.237123:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948685.237141:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000100:00000200:1.0:1784948685.237162:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289645568, portal 10 00000100:00000001:1.0:1784948685.237180:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948685.237187:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558293120 : -115544151258496 : ffff96e9c7e88a80) 00000100:00000001:3.0:1784948685.237188:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948685.237210:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948685.237216:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948685.237324:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.237325:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948685.237342:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948685.237342:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:1.0:1784948685.237362:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000100:00000001:3.0:1784948685.237370:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948685.237373:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289645568, offset 0 00000100:00000001:3.0:1784948685.237387:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305624619 00000400:00000200:1.0:1784948685.237401:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000010:3.0:1784948685.237407:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:3.0:1784948685.237425:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000200:1.0:1784948685.237436:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784948685.237444:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305624619 00000400:00000200:1.0:1784948685.237454:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:3.0:1784948685.237459:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000400:00000200:1.0:1784948685.237474:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948685.237481:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000400:00000200:1.0:1784948685.237490:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948685.237503:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305624619 00000400:00000200:1.0:1784948685.237506:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000010:3.0:1784948685.237526:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000400:00000200:1.0:1784948685.237534:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3117:3117:236:4294967295] 192.168.202.137@tcp LPNI seq info [2025:2025:8:4294967295] 00000100:00000001:3.0:1784948685.237545:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948685.237597:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305624619 00000400:00000200:1.0:1784948685.237611:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000010:3.0:1784948685.237619:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:3.0:1784948685.237646:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000800:00000200:1.0:1784948685.237655:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948685.237668:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1784948685.237670:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000100:00000001:3.0:1784948685.237687:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948685.237705:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948685.237707:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948685.237726:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948685.237743:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784948685.237753:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:1.0:1784948685.237755:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948685.237761:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948685.237774:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948685.237782:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948685.237788:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948685.237790:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948685.237801:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2012 usecs (3 jiffies) 00000100:00000001:3.0:1784948685.237801:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.237820:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305624580 00000100:00000001:1.0:1784948685.237830:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305624619 00000100:00000001:1.0:1784948685.237838:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305624580 00000100:00000001:1.0:1784948685.237848:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305624619 00000100:00000001:1.0:1784948685.237859:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305624580 00000100:00000001:1.0:1784948685.237869:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305624619 00000100:00000001:1.0:1784948685.237878:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305624580 00000100:00000001:1.0:1784948685.237892:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305624619 00000100:00000001:1.0:1784948685.237901:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948685.237968:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.237979:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.237989:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.238003:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948685.238010:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948685.238019:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948685.238027:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948685.238032:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.238040:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948685.238098:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948685.238109:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000800:00000001:0.0:1784948685.238127:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948685.238130:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948685.238149:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948685.238151:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948685.238171:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948685.238175:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000800:00000001:0.0:1784948685.238182:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948685.238187:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000100:00000001:3.0:1784948685.238196:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784948685.238202:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:3.0:1784948685.238212:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948685.238228:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434200 00000100:00000001:3.0:1784948685.238230:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948685.238246:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948685.238266:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 147984 00000100:00000001:1.0:1784948685.238272:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948685.238286:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:0.0:1784948685.238294:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948685.238305:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948685.238308:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:0.0:1784948685.238310:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948685.238317:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948685.238322:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000001:1.0:1784948685.238322:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948685.238338:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000400:00000200:0.0:1784948685.238355:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948685.238358:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000400:00000010:1.0:1784948685.238368:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000100:00000001:1.0:1784948685.238380:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948685.238382:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948685.238393:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:1.0:1784948685.238395:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784948685.238413:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004c9f9b28. 00000100:00000040:0.0:1784948685.238428:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000004c9f9b28 x1871651289645568 msgsize 224 00000100:00000200:1.0:1784948685.238443:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948685.238469:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948685.238489:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948685.238495:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948685.238505:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948685.238510:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948685.238515:0:337120:0:(client.c:1580:after_reply()) Process entered 00000800:00000001:0.0:1784948685.238521:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948685.238525:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948685.238533:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948685.238542:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948685.238552:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948685.238583:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948685.238590:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289645568 00000100:00001000:1.0:1784948685.238591:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=7 v=5 (5 5 5 5) 02000000:00000001:3.0:1784948685.238597:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948685.238603:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00001000:1.0:1784948685.238610:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=7 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948685.238612:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948685.238621:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.238625:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00100000:3.0:1784948685.238629:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289645568 00000100:00000001:1.0:1784948685.238631:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948685.238635:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948685.238643:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000001:1.0:1784948685.238644:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784948685.238647:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00010000:00000001:1.0:1784948685.238655:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948685.238655:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948685.238669:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948685.238672:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948685.238675:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948685.238687:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784948685.238693:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948685.238693:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000056fd134f. 00000100:00000001:3.0:1784948685.238704:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948685.238716:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000056fd134f. 00000020:00000010:3.0:1784948685.238734:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a7af9ab9. 00000100:00000001:1.0:1784948685.238740:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948685.238743:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dc76901d. 00000100:00000001:1.0:1784948685.238754:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:3.0:1784948685.238756:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000e923a07a. 00000100:00000001:1.0:1784948685.238761:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948685.238773:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:1.0:1784948685.238780:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948685.238782:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948685.238788:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948685.238793:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948685.238807:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948685.238812:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00100000:1.0:1784948685.238813:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000c8b06f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289645440:0@lo:400: 00000100:00000001:3.0:1784948685.238816:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.238832:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.238836:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948685.238841:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948685.238843:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000c8b06f4 x1871651289645440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948685.238856:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948685.238860:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784948685.238861:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:1.0:1784948685.238870:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:3.0:1784948685.238871:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 656 02000000:00000010:1.0:1784948685.238880:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000005e186afc. 00000100:00000040:3.0:1784948685.238882:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948685.238892:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558299392 : -115544151252224 : ffff96e9c7e8a300) 02000000:00000001:1.0:1784948685.238902:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784948685.238943:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000004c9f9b28 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:346/0 lens 224/0 e 0 to 0 dl 1784948696 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948685.238958:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:1.0:1784948685.238967:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:3.0:1784948685.238967:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000056fd134f. 00000100:00000001:3.0:1784948685.238977:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784948685.238978:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000010:3.0:1784948685.238984:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000056fd134f. 00000020:00000001:1.0:1784948685.238995:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00100000:3.0:1784948685.239003:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000004c9f9b28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289645568:12345-192.168.202.137@tcp:400: 02000000:00000010:1.0:1784948685.239005:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a3cf4688. 00000100:00000200:3.0:1784948685.239019:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289645568 02000000:00000001:1.0:1784948685.239026:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1784948685.239026:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:1.0:1784948685.239036:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1784948685.239037:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948685.239044:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948685.239046:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000c8b06f4. 00000020:00000001:3.0:1784948685.239055:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948685.239058:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784948685.239061:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1784948685.239064:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784948685.239067:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948685.239070:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.239083:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948685.239084:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948685.239089:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948685.239098:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948685.239102:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948685.239104:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948685.239109:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948685.239110:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948685.239114:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948685.239122:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784948685.239123:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007da3ec17. 02000000:00000001:3.0:1784948685.239131:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.239132:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948685.239144:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948685.239149:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948685.239153:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289645568 00010000:00000001:3.0:1784948685.239156:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948685.239163:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000004c9f9b28 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:346/0 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948685.239174:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948685.239177:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948685.239182:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=152 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948685.239188:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948685.239193:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948685.239197:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784948685.239202:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948685.239205:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948685.239211:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948685.239215:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948685.239221:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000200:3.0:1784948685.239229:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289645568, offset 224 00000400:00000200:3.0:1784948685.239238:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948685.239254:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948685.239265:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3118:3118:236:4294967295] 192.168.202.137@tcp LPNI seq info [2026:2026:8:4294967295] 00000400:00000200:3.0:1784948685.239284:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948685.239295:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948685.239303:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003264a2f8. 00000800:00000200:3.0:1784948685.239314:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948685.239329:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948685.239337:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003264a2f8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948685.239362:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948685.239369:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948685.239374:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948685.239378:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948685.239382:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948685.239392:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004c9f9b28 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:346/0 lens 224/224 e 0 to 0 dl 1784948696 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948685.239415:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000004c9f9b28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289645568:12345-192.168.202.137@tcp:400: Request processed in 451us (981us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948685.239431:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 656 00000100:00000040:3.0:1784948685.239438:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948685.239443:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948685.239448:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948685.239455:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dc76901d. 00000020:00000010:3.0:1784948685.239464:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000e923a07a. 00000020:00000010:3.0:1784948685.239471:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a7af9ab9. 00000020:00000040:3.0:1784948685.239483:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948685.239490:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948685.239620:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948685.239642:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948685.239651:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948685.239660:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948685.239674:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948685.239690:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434200 00000400:00000200:3.0:1784948685.239703:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x7f89 [1] + 224 00000800:00000001:3.0:1784948685.239712:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948685.239725:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948685.239730:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948685.239738:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948685.239747:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948685.239754:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948685.239773:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948685.239809:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948685.239820:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948685.239825:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.239874:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948685.239880:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948685.239886:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948685.239889:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948685.239893:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dbb46726. 00000400:00000200:1.0:1784948685.239899:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:1.0:1784948685.239902:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000100:00000001:1.0:1784948685.239944:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948685.239953:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948685.239967:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948685.239979:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948685.239982:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.239984:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948685.239987:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948685.239989:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948685.239991:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948685.239995:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948685.240002:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=152 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948685.240008:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=152 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948685.240011:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948685.240013:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948685.240016:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948685.240022:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948685.240044:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.240048:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948685.240051:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948685.240054:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948685.240056:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948685.240061:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948685.240071:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948685.240075:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948685.240077:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948685.240081:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948685.240092:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ee0ebd63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289645568:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948685.240099:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.240102:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948685.240106:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ee0ebd63 x1871651289645568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948685.240133:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948685.240136:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948685.240139:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000da755428. 02000000:00000001:1.0:1784948685.240144:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948685.240147:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948685.240149:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948685.240153:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948685.240155:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f64a2f6c. 02000000:00000001:1.0:1784948685.240160:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948685.240162:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948685.240164:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ee0ebd63. 00000100:00000001:1.0:1784948685.240169:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948685.240171:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.240174:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948685.240182:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948685.240184:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948685.240188:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948685.240191:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948685.280009:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305624619, at 4305624620 00000100:00000010:0.0:1784948685.280022:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:3.0:1784948685.280048:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305624619, at 4305624620 00000100:00000040:1.0:1784948685.280054:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305624619, at 4305624620 00000100:00000010:3.0:1784948685.280066:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00000100:00000010:1.0:1784948685.280083:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000400:00000100:3.0:1784948685.748170:0:343755:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.15:988 00000400:00000100:1.0:1784948685.748259:0:343654:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.24:988 00000400:00000100:0.0:1784948685.748282:0:343953:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.3:988 00000400:00000100:3.0:1784948685.748313:0:343755:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.15@tcp at host 111.222.173.15:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948685.748335:0:343755:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.15@tcp: down 00000400:00000200:3.0:1784948685.748348:0:343755:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1784948685.748363:0:343755:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:3.0:1784948685.748373:0:343755:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f8a82730. 00000400:00000200:3.0:1784948685.748387:0:343755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948685.748402:0:343755:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000100:1.0:1784948685.748411:0:343654:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.24@tcp at host 111.222.173.24:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948685.748419:0:343755:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 10958 health :900 00000400:00000200:3.0:1784948685.748432:0:343755:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000100:0.0:1784948685.748433:0:343953:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.3@tcp at host 111.222.173.3:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948685.748446:0:343654:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.24@tcp: down 00000400:00000200:3.0:1784948685.748449:0:343755:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:0.0:1784948685.748475:0:343953:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.3@tcp: down 00000400:00000200:3.0:1784948685.748487:0:343755:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:0.0:1784948685.748531:0:343953:0:(router.c:1888:lnet_notify()) Old news 00000800:00000200:1.0:1784948685.748533:0:343654:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.173.24@tcp down 00000800:00000100:0.0:1784948685.748546:0:343953:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000800:00000010:0.0:1784948685.748553:0:343953:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006ee15577. 00000400:00000200:0.0:1784948685.748587:0:343953:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948685.748605:0:343953:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948685.748617:0:343953:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948685.748628:0:343953:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:0.0:1784948685.748645:0:343953:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000100:1.0:1784948685.748649:0:343958:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.7:988 00000400:00000100:3.0:1784948685.748659:0:343955:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.5:988 00000400:00000100:0.0:1784948685.748726:0:343914:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.1:988 00000400:00000100:1.0:1784948685.748752:0:343958:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.7@tcp at host 111.222.173.7:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784948685.748764:0:343955:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.5@tcp at host 111.222.173.5:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948685.748779:0:343958:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.7@tcp: down 00000400:00000200:1.0:1784948685.748792:0:343958:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:0.0:1784948685.748794:0:343914:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.1@tcp at host 111.222.173.1:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948685.748794:0:343955:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.5@tcp: down 00000400:00000200:3.0:1784948685.748813:0:343955:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784948685.748820:0:343958:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000800:00000010:1.0:1784948685.748830:0:343958:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000aae658b7. 00000400:00000200:0.0:1784948685.748832:0:343914:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.1@tcp: down 00000400:00000200:0.0:1784948685.748852:0:343914:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784948685.748853:0:343958:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948685.748882:0:343958:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948685.748902:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784948685.748971:0:343958:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948685.748980:0:343958:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:3.0:1784948685.749006:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000200:1.0:1784948685.749014:0:343958:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000001:3.0:1784948685.749026:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948685.749032:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948685.749037:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdaa2431 00000400:00000010:3.0:1784948685.749055:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdaa2431. 00000400:00000200:3.0:1784948685.749063:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948685.749161:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:3.0:1784948685.749174:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eff166f6. 00000400:00000200:3.0:1784948685.749207:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:3.0:1784948685.749225:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:3.0:1784948685.749246:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000400:00000200:3.0:1784948685.749267:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:3.0:1784948685.749290:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948685.749297:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948685.749303:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000400:00000010:3.0:1784948685.749306:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000400:00000100:1.0:1784948685.749308:0:343956:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.13:988 00000400:00000200:3.0:1784948685.749319:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948685.749339:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:3.0:1784948685.749343:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032fe57fd. 00000400:00000200:3.0:1784948685.749356:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:3.0:1784948685.749368:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:3.0:1784948685.749380:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:3.0:1784948685.749401:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000100:1.0:1784948685.749405:0:343956:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.13@tcp at host 111.222.173.13:988 took too long: that node may be hung or experiencing high load. 00000400:00000001:3.0:1784948685.749408:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948685.749417:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948685.749421:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000400:00000200:1.0:1784948685.749429:0:343956:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.13@tcp: down 00000400:00000010:3.0:1784948685.749431:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000400:00000200:1.0:1784948685.749441:0:343956:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784948685.749443:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948685.749455:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000800:00000100:1.0:1784948685.749457:0:343956:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000800:00000010:1.0:1784948685.749474:0:343956:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000595fa8f8. 00000400:00000010:3.0:1784948685.749475:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005152bcfe. 00000400:00000200:1.0:1784948685.749506:0:343956:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948685.749511:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:3.0:1784948685.749525:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:1.0:1784948685.749530:0:343956:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948685.749543:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000400:00000200:1.0:1784948685.749556:0:343956:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948685.749557:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000200:1.0:1784948685.749596:0:343956:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000001:3.0:1784948685.749597:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948685.749607:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948685.749615:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000400:00000200:1.0:1784948685.749622:0:343956:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:3.0:1784948685.749624:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000400:00000200:3.0:1784948685.749638:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948685.749652:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:3.0:1784948685.749660:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f11303f8. 00000400:00000200:3.0:1784948685.749666:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:3.0:1784948685.749669:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:3.0:1784948685.749673:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000400:00000200:3.0:1784948685.749678:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:3.0:1784948685.749680:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948685.749681:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948685.749683:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:3.0:1784948685.749685:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000400:00000200:3.0:1784948685.749687:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948685.749690:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:3.0:1784948685.749692:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000859865aa. 00000400:00000200:3.0:1784948685.749698:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:3.0:1784948685.749701:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:3.0:1784948685.749704:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000020:00000001:3.0:1784948686.068166:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948686.068176:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948686.068182:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:1.0:1784948686.452089:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948686.452108:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000eb33d71d. 00000400:00000010:1.0:1784948686.452126:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000da755428. 00000400:00000010:1.0:1784948686.452135:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000400:00000010:1.0:1784948686.452147:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f65f2fb7. 00000400:00000200:1.0:1784948686.452154:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948686.452180:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948686.452190:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948686.452196:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3119:3119:241:4294967295] 192.168.202.137@tcp LPNI seq info [2027:2027:8:4294967295] 00000400:00000200:1.0:1784948686.452207:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948686.452218:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948686.452225:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948686.452233:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002a753d9d. 00000800:00000200:1.0:1784948686.452243:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948686.452257:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948686.452264:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002a753d9d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948686.452293:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 10960 health :500 00000800:00000200:0.0:1784948686.452509:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948686.452523:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002a753d9d. 00000400:00000200:0.0:1784948686.452538:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948686.452581:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948686.452593:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948686.452599:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948686.452986:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948686.453198:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948687.092292:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948687.092300:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948687.092305:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948688.116174:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948688.116179:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948688.116181:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948689.140247:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948689.140256:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948689.140262:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948689.332656:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948689.332667:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948689.332677:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b47dc450. 00000020:00000040:1.0:1784948689.332685:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948689.332692:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948689.332696:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948689.332704:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948689.332716:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003ed79484. 00000100:00000001:1.0:1784948689.332726:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.332735:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948689.332750:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948689.332754:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.332761:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b47dc450] to pc [ptlrpcd_00_03+3] req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.332798:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948689.332801:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948689.332807:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948689.332811:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948689.332813:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948689.332819:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784948689.332825:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10936 00000100:00000001:0.0:1784948689.332828:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948689.332838:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948689.332844:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:0.0:1784948689.332847:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1784948689.332856:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 10935 00000100:00000040:0.0:1784948689.332861:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948689.332880:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:0.0:1784948689.332891:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784948689.332892:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:0.0:1784948689.332898:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.332942:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948689.332952:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784948689.332955:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784948689.332961:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:0.0:1784948689.332962:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948689.332971:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:0.0:1784948689.332972:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948689.332982:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000100:00100000:0.0:1784948689.332987:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b47dc450 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289645696:0@lo:400: 00000020:00000040:1.0:1784948689.332989:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:1.0:1784948689.332996:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948689.333000:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1784948689.333004:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000200:3.0:1784948689.333008:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000001:1.0:1784948689.333010:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:0.0:1784948689.333012:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948689.333021:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948689.333022:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:0.0:1784948689.333028:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948689.333029:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000040:1.0:1784948689.333034:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000010:0.0:1784948689.333041:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb983bef. 00000100:00000001:1.0:1784948689.333042:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1784948689.333048:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a7af9ab9. 02000000:00000001:0.0:1784948689.333050:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.333052:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948689.333061:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:3.0:1784948689.333063:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003dcd3ddc. 00000020:00000040:1.0:1784948689.333066:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:0.0:1784948689.333067:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000313d28ec. 00000100:00080000:1.0:1784948689.333079:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000010:3.0:1784948689.333080:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008a1e1d5c. 00000400:00000010:0.0:1784948689.333084:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000100:00080000:1.0:1784948689.333087:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000200:3.0:1784948689.333096:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:1.0:1784948689.333099:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948689.333104:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1784948689.333108:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289645696, portal 25 00000100:00000001:0.0:1784948689.333116:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1784948689.333117:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948689.333123:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:0.0:1784948689.333124:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451656832 : -115544257894784 : ffff96e9c18d6680) 02000000:00000001:1.0:1784948689.333125:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.333138:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:0.0:1784948689.333144:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948689.333145:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784948689.333146:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948689.333160:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005e9e18c4. 00000400:00000200:3.0:1784948689.333161:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000040:1.0:1784948689.333170:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:3.0:1784948689.333178:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948689.333181:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784948689.333183:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1784948689.333188:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000040:0.0:1784948689.333191:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784948689.333191:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.333202:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1784948689.333203:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3121:3121:241:4294967295] 1.2.3.4@tcp LPNI seq info [93:93:8:4294967295] 00000400:00000010:0.0:1784948689.333204:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 02000000:00000001:1.0:1784948689.333207:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948689.333214:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948689.333217:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289645696, offset 0 02000000:00000010:1.0:1784948689.333226:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000056522212. 00000400:00000200:3.0:1784948689.333227:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000400:00000200:0.0:1784948689.333236:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948689.333239:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948689.333253:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00100000:1.0:1784948689.333258:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005e9e18c4 x1871651289645824/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1784948689.333275:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000400:00000200:0.0:1784948689.333284:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:3.0:1784948689.333292:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000751c5583. 00000100:00000001:1.0:1784948689.333302:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948689.333305:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948689.333315:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:1.0:1784948689.333317:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e9e18c4] to pc [ptlrpcd_rcv+-1] req@000000005e9e18c4 x1871651289645824/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948689.333318:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434280 00000800:00000200:3.1:1784948689.333333:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:0.0:1784948689.333341:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7591 [2] + 0 00000400:00000010:0.0:1784948689.333353:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b69fa69e. 00000400:00000200:0.0:1784948689.333362:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000043dfa704 00000100:00000001:1.0:1784948689.333365:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.333373:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1784948689.333374:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000400:00000200:0.0:1784948689.333380:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948689.333384:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:3.0:1784948689.333386:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 00000100:00000010:1.0:1784948689.333394:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004f710a78. 00000400:00000200:0.0:1784948689.333400:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000040:1.0:1784948689.333403:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:0.0:1784948689.333409:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000043dfa704 00000100:00000001:1.0:1784948689.333411:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1784948689.333416:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000043dfa704. 02000000:00000001:1.0:1784948689.333417:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948689.333425:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.333429:0:337122:0:(events.c:286:request_in_callback()) Process entered 02000000:00000010:1.0:1784948689.333433:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000200:0.0:1784948689.333434:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784948689.333440:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948689.333444:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000097ccad2 x1871651289645696 msgsize 224 00000100:00000040:1.0:1784948689.333452:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1784948689.333469:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948689.333480:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948689.333481:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948689.333487:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.333498:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004f710a78] to pc [ptlrpcd_00_00+0] req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948689.333506:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948689.333520:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:0.0:1784948689.333527:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00000001:0.0:1784948689.333568:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948689.333580:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.333586:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.333602:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10966) 00000100:00000001:0.0:1784948689.333613:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948689.333615:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:0.0:1784948689.333624:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948689.333628:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289645696 00000100:00000001:1.0:1784948689.333630:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948689.333639:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784948689.333641:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948689.333647:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948689.333650:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948689.333659:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.333661:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948689.333676:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.333677:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.333691:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1784948689.333694:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289645696 00000100:00000001:0.0:1784948689.333704:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948689.333704:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948689.333712:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784948689.333714:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1784948689.333719:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948689.333720:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305628715 00000100:00000001:1.0:1784948689.333721:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948689.333728:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 02000000:00000001:1.0:1784948689.333734:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:0.0:1784948689.333736:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000020:00000001:3.0:1784948689.333739:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:1.0:1784948689.333740:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948689.333752:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1784948689.333753:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:1.0:1784948689.333756:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948689.333769:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:1.0:1784948689.333772:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004f710a78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289645952:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948689.333773:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305628715 00000020:00000010:3.0:1784948689.333779:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d92bf133. 00000100:00000010:0.0:1784948689.333787:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:1.0:1784948689.333791:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948689.333792:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948689.333797:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1784948689.333801:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000010:3.0:1784948689.333801:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d92bf133. 00000100:00000001:0.0:1784948689.333807:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305628715 02000000:00000001:1.0:1784948689.333808:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948689.333817:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:3.0:1784948689.333819:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000100:00000010:0.0:1784948689.333822:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 02000000:00000010:1.0:1784948689.333827:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000002c1a3d41. 00000020:00000010:3.0:1784948689.333835:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dc76901d. 00000100:00000001:0.0:1784948689.333836:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:1.0:1784948689.333837:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.333850:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305628715 00000020:00000010:3.0:1784948689.333852:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000e923a07a. 00000400:00000010:1.0:1784948689.333855:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dbb46726. 00000100:00000010:0.0:1784948689.333861:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:0.0:1784948689.333869:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000010:1.0:1784948689.333875:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000040:3.0:1784948689.333875:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948689.333877:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948689.333886:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948689.333889:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948689.333893:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289645952, portal 10 00000100:00000001:3.0:1784948689.333894:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948689.333895:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948689.333901:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.333944:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.333948:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948689.333956:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948689.333956:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451648768 : -115544257902848 : ffff96e9c18d4700) 00000100:00000001:0.0:1784948689.333962:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948689.333965:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.333969:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948689.333974:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948689.333978:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948689.333980:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948689.333987:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948689.333989:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.334006:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948689.334014:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948689.334019:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000100:00000001:3.0:1784948689.334023:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000200:1.0:1784948689.334025:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289645952, offset 0 00000100:00000001:3.0:1784948689.334030:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784948689.334039:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:3.0:1784948689.334041:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 681 00000100:00000040:3.0:1784948689.334054:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:1.0:1784948689.334064:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784948689.334064:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466820688 : -115544242730928 : ffff96e9c274c850) 00000400:00000200:1.0:1784948689.334085:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1784948689.334086:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000097ccad2 x1871651289645696/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:350/0 lens 224/0 e 0 to 0 dl 1784948700 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948689.334097:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948689.334109:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948689.334115:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784948689.334116:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:3.0:1784948689.334123:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d92bf133. 00000400:00000200:1.0:1784948689.334125:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3122:3122:240:4294967295] 192.168.202.137@tcp LPNI seq info [2029:2029:8:4294967295] 00000100:00000001:3.0:1784948689.334134:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948689.334140:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d92bf133. 00000400:00000200:1.0:1784948689.334153:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:3.0:1784948689.334156:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000097ccad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289645696:12345-0@lo:400: 00000100:00000200:3.0:1784948689.334168:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289645696 00000800:00000200:1.0:1784948689.334173:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784948689.334176:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000010:1.0:1784948689.334182:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000662152ce. 00000020:00000001:3.0:1784948689.334184:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948689.334192:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948689.334200:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:3.0:1784948689.334201:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948689.334207:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784948689.334214:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948689.334217:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948689.334222:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:3.0:1784948689.334225:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000200:1.0:1784948689.334233:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000662152ce type 1, nob 320 niov 1 nkiov 1 00000020:00000001:3.0:1784948689.334234:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948689.334240:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948689.334245:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948689.334249:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948689.334252:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948689.334259:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fd4fe1c8. 00000100:00000001:1.0:1784948689.334260:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948689.334265:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.334271:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2016 usecs (3 jiffies) 00000100:00000001:3.0:1784948689.334271:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948689.334281:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.334282:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305628677 00010000:00000040:3.0:1784948689.334292:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289645696 00000100:00000001:1.0:1784948689.334293:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305628715 00000100:00000001:1.0:1784948689.334304:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305628677 00010000:00000001:3.0:1784948689.334305:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784948689.334314:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305628715 00010000:00000200:3.0:1784948689.334319:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000097ccad2 x1871651289645696/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:350/0 lens 224/224 e 0 to 0 dl 1784948700 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.334321:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305628677 00000100:00000001:1.0:1784948689.334331:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305628715 00000100:00000001:1.0:1784948689.334340:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305628677 00000100:00000001:1.0:1784948689.334350:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305628715 00010000:00000001:3.0:1784948689.334352:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784948689.334355:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784948689.334358:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.334361:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948689.334365:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=11 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948689.334368:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948689.334374:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784948689.334375:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948689.334382:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948689.334386:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784948689.334387:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784948689.334390:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784948689.334393:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 02000000:00000001:3.0:1784948689.334395:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.334400:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948689.334402:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948689.334405:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948689.334412:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784948689.334413:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948689.334420:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000100:00000001:1.0:1784948689.334421:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948689.334432:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289645696, offset 224 00000400:00000200:3.0:1784948689.334441:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948689.334457:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:0.0:1784948689.334462:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1784948689.334471:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434280 00000800:00000010:0.0:1784948689.334473:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000662152ce. 00000400:00000200:0.0:1784948689.334484:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948689.334490:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7fa5 [1] + 224 00000400:00000200:0.0:1784948689.334500:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948689.334511:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:0.0:1784948689.334516:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000400:00000200:3.0:1784948689.334516:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948689.334526:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1784948689.334558:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:0.0:1784948689.334570:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948689.334573:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948689.334589:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948689.334596:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948689.334606:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948689.334647:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948689.334656:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.334667:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948689.334698:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1784948689.334731:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:3.0:1784948689.334737:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000100:00000001:3.0:1784948689.334745:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948689.334750:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948689.334752:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948689.334759:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948689.334762:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948689.334769:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1784948689.334770:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:1.0:1784948689.334778:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:3.0:1784948689.334779:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:1.0:1784948689.334786:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000313d28ec. 00000020:00000001:3.0:1784948689.334787:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948689.334800:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000020:00000001:3.0:1784948689.334800:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:1.0:1784948689.334810:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000100:00000040:3.0:1784948689.334815:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000097ccad2 x1871651289645696/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:350/0 lens 224/224 e 0 to 0 dl 1784948700 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.334821:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948689.334834:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948689.334864:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000097ccad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289645696:12345-0@lo:400: Request processed in 741us (1411us total) trans 0 rc 0/0 00000100:00000200:1.0:1784948689.334870:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.334895:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1784948689.334900:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 681 00000100:00000001:1.0:1784948689.334945:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.334951:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:3.0:1784948689.334959:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 02000000:00000001:1.0:1784948689.334960:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948689.334963:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948689.334966:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948689.334968:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00100000:0.0:1784948689.334970:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289645952 00000100:00000001:1.0:1784948689.334971:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948689.334975:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:0.0:1784948689.334980:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:1.0:1784948689.334982:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948689.334986:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dc76901d. 00000100:00000001:0.0:1784948689.334988:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00001000:1.0:1784948689.334991:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=11 v=5 (5 5 5 5) 00000100:00000001:0.0:1784948689.334995:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948689.335000:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000e923a07a. 00000100:00001000:1.0:1784948689.335002:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=11 v=5 (1 1 1 1) 02000000:00000001:0.0:1784948689.335005:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948689.335011:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000100:00100000:0.0:1784948689.335017:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289645952 00000100:00000001:1.0:1784948689.335018:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000040:3.0:1784948689.335024:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784948689.335027:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948689.335028:0:342764:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:3.0:1784948689.335033:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000040:0.0:1784948689.335034:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000001:1.0:1784948689.335036:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000010:3.0:1784948689.335040:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000fd4fe1c8. 00000020:00000001:0.0:1784948689.335042:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00010000:00000001:1.0:1784948689.335044:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948689.335051:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000040:0.0:1784948689.335052:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:1.0:1784948689.335052:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948689.335059:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948689.335063:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:1.0:1784948689.335068:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948689.335069:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:0.0:1784948689.335071:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948689.335083:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:3.0:1784948689.335084:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001004a12d. 00000020:00000010:0.0:1784948689.335092:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:1.0:1784948689.335095:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948689.335099:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784948689.335101:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006c7a4b68. 00000100:00000001:1.0:1784948689.335102:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:0.0:1784948689.335105:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:1.0:1784948689.335108:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948689.335119:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948689.335122:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000020:00000010:0.0:1784948689.335134:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008cc004d6. 00000100:00100000:1.0:1784948689.335140:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b47dc450 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289645696:0@lo:400: 00000020:00000010:0.0:1784948689.335144:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784948689.335150:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.335156:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948689.335160:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:1.0:1784948689.335161:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b47dc450 x1871651289645696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948689.335169:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948689.335175:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948689.335181:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.335185:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948689.335191:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1784948689.335192:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000010:1.0:1784948689.335194:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb983bef. 00000100:00000001:0.0:1784948689.335199:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:1.0:1784948689.335203:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784948689.335204:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948689.335210:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948689.335213:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784948689.335217:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948689.335219:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003ed79484. 02000000:00000001:1.0:1784948689.335222:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948689.335224:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948689.335226:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b47dc450. 00000100:00000001:0.0:1784948689.335227:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.335234:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948689.335239:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.335242:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948689.335246:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948689.335249:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948689.335252:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948689.335255:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.335260:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948689.335264:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 657 00000100:00000001:1.0:1784948689.335266:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948689.335275:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948689.335280:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470017920 : -115544239533696 : ffff96e9c2a59180) 00000100:00000040:0.0:1784948689.335287:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000887a7a1e x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:350/0 lens 224/0 e 0 to 0 dl 1784948700 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948689.335300:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948689.335305:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:0.0:1784948689.335308:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948689.335310:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00100000:0.0:1784948689.335316:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000887a7a1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289645952:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948689.335321:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289645952 00000020:00000001:0.0:1784948689.335324:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948689.335328:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948689.335330:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948689.335333:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948689.335336:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948689.335339:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948689.335342:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948689.335345:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948689.335347:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948689.335349:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948689.335351:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948689.335356:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948689.335358:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948689.335363:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000004469f10a. 02000000:00000001:0.0:1784948689.335366:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.335369:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948689.335371:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948689.335374:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289645952 00010000:00000001:0.0:1784948689.335377:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948689.335381:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000887a7a1e x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:350/0 lens 224/224 e 0 to 0 dl 1784948700 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948689.335391:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948689.335393:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948689.335396:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=6 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948689.335400:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948689.335403:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948689.335406:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948689.335409:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948689.335412:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948689.335415:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948689.335418:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948689.335421:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000100:00000200:0.0:1784948689.335425:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289645952, offset 224 00000400:00000200:0.0:1784948689.335430:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948689.335440:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948689.335447:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3123:3123:240:4294967295] 192.168.202.137@tcp LPNI seq info [2030:2030:8:4294967295] 00000400:00000200:0.0:1784948689.335459:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948689.335465:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948689.335471:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b84187a1. 00000800:00000200:0.0:1784948689.335476:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948689.335483:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948689.335487:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b84187a1 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948689.335506:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948689.335510:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948689.335513:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948689.335515:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948689.335518:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948689.335523:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000887a7a1e x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:350/0 lens 224/224 e 0 to 0 dl 1784948700 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948689.335567:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000887a7a1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289645952:12345-192.168.202.137@tcp:400: Request processed in 265us (763us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948689.335576:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 657 00000100:00000040:0.0:1784948689.335580:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948689.335584:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948689.335587:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948689.335591:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008cc004d6. 00000020:00000010:0.0:1784948689.335595:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948689.335599:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000020:00000040:0.0:1784948689.335606:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948689.335610:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948689.335681:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948689.335696:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948689.335707:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948689.335717:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948689.335732:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948689.335750:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434380 00000400:00000200:0.0:1784948689.335758:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x7fad [1] + 224 00000800:00000001:0.0:1784948689.335767:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948689.335779:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948689.335783:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948689.335788:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948689.335793:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948689.335798:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948689.335809:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948689.335831:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948689.335839:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948689.335844:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.335938:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948689.335944:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948689.335952:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948689.335954:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948689.335957:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dbb46726. 00000400:00000200:1.0:1784948689.335964:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784948689.335966:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784948689.335970:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948689.335977:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948689.335990:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.336001:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948689.336003:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.336006:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948689.336008:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948689.336010:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948689.336012:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948689.336015:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948689.336021:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=6 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948689.336025:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=6 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948689.336028:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948689.336030:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948689.336032:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948689.336037:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948689.336046:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.336050:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948689.336052:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948689.336054:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948689.336055:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948689.336059:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.336067:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948689.336070:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948689.336072:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948689.336076:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948689.336086:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004f710a78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289645952:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948689.336091:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.336093:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948689.336095:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004f710a78 x1871651289645952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948705 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948689.336103:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948689.336105:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948689.336107:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000002c1a3d41. 02000000:00000001:1.0:1784948689.336111:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948689.336113:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948689.336114:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948689.336117:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948689.336118:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:1.0:1784948689.336121:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948689.336122:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948689.336124:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004f710a78. 00000100:00000001:1.0:1784948689.336126:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948689.336128:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.336130:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948689.336135:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948689.336137:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948689.336140:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948689.336142:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948689.376032:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305628715, at 4305628716 00000100:00000010:0.0:1784948689.376049:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000040:1.0:1784948689.376051:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305628715, at 4305628716 00000100:00000040:3.0:1784948689.376067:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305628715, at 4305628716 00000100:00000010:1.0:1784948689.376068:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:3.0:1784948689.376091:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000020:00000001:3.0:1784948690.164238:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948690.164248:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948690.164255:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1784948690.356329:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948690.356346:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784948690.356364:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000035b51214. 00000020:00000040:3.0:1784948690.356380:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948690.356398:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948690.356410:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948690.356423:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948690.356436:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000004805f207. 00000100:00000001:3.0:1784948690.356451:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.356462:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948690.356482:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948690.356487:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.356498:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035b51214] to pc [ptlrpcd_00_01+1] req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948690.356603:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948690.356610:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948690.356618:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948690.356624:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784948690.356629:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784948690.356640:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784948690.356659:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10936 00000100:00000001:1.0:1784948690.356660:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948690.356669:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948690.356690:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:3.0:1784948690.356706:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 10961 00000400:00000200:3.0:1784948690.356731:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:3.0:1784948690.356741:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1784948690.356750:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948690.356758:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948690.356769:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948690.356776:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:3.0:1784948690.356783:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948690.356790:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:3.0:1784948690.356797:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948690.356803:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:3.0:1784948690.356811:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:3.0:1784948690.356815:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:3.0:1784948690.356821:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948690.356838:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948690.356848:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948690.356859:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:3.0:1784948690.356868:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:3.0:1784948690.356885:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00080000:3.0:1784948690.356896:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:3.0:1784948690.356953:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948690.356959:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1784948690.356973:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948690.356981:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.356984:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1784948690.356989:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948690.357001:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000adacbaf0. 00000020:00000040:3.0:1784948690.357008:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784948690.357015:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784948690.357021:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.357035:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948690.357038:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948690.357043:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948690.357055:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000fd4fe1c8. 00000100:00000001:3.0:1784948690.357067:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948690.357083:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948690.357107:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948690.357114:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.357122:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adacbaf0] to pc [ptlrpcd_rcv+-1] req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948690.357168:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.357175:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948690.357184:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784948690.357189:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fa1e143a. 00000020:00000040:3.0:1784948690.357198:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948690.357209:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948690.357218:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948690.357221:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948690.357224:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948690.357229:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1784948690.357241:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000040401747. 00000100:00000001:3.0:1784948690.357251:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.357265:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948690.357298:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948690.357304:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.357319:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fa1e143a] to pc [ptlrpcd_00_02+2] req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948690.357338:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948690.357356:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948690.357370:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.357386:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10967) 00000100:00000001:1.0:1784948690.357410:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948690.357414:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.357415:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948690.357422:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948690.357424:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784948690.357429:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948690.357430:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784948690.357440:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948690.357446:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948690.357459:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000adacbaf0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289646208:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784948690.357478:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948690.357482:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1784948690.357487:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948690.357489:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948690.357492:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948690.357496:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948690.357504:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1784948690.357506:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1784948690.357517:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000371d55ed. 02000000:00000001:3.0:1784948690.357518:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948690.357549:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948690.357565:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dbb46726. 00000100:00100000:3.0:1784948690.357565:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fa1e143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289646336:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948690.357580:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:1.0:1784948690.357584:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 02000000:00000001:3.0:1784948690.357587:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784948690.357591:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:3.0:1784948690.357596:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948690.357598:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289646208, portal 4 00000100:00100000:0.0:1784948690.357599:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289646080 02000000:00000001:3.0:1784948690.357607:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948690.357611:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1784948690.357612:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000010:3.0:1784948690.357615:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005e186afc. 00000100:00000001:1.0:1784948690.357617:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482528768 : -115544227022848 : ffff96e9c3647800) 00000100:00000001:0.0:1784948690.357619:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:3.0:1784948690.357625:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948690.357636:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948715 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948690.357637:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b69fa69e. 00000400:00000010:3.0:1784948690.357650:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000100:00000001:1.0:1784948690.357664:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:3.0:1784948690.357665:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289646336, portal 10 00000100:00000001:3.0:1784948690.357673:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1784948690.357675:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948690.357679:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.357681:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482520704 : -115544227030912 : ffff96e9c3645880) 00000400:00000010:1.0:1784948690.357686:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 02000000:00000001:0.0:1784948690.357691:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948690.357694:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289646208, offset 0 00000100:00100000:0.0:1784948690.357701:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289646080 00000100:00000040:3.0:1784948690.357702:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948690.357711:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948690.357717:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:1.0:1784948690.357719:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784948690.357727:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948690.357735:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948690.357740:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784948690.357745:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:0.0:1784948690.357746:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948690.357756:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:3.0:1784948690.357759:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948690.357764:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784948690.357769:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:3.0:1784948690.357770:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000400:00000200:1.0:1784948690.357776:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000200:3.0:1784948690.357781:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289646336, offset 0 00000400:00000200:1.0:1784948690.357787:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1784948690.357798:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000400:00000200:1.0:1784948690.357799:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948690.357800:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948690.357810:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948690.357816:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3124:3124:240:4294967295] 192.168.202.137@tcp LPNI seq info [2031:2031:8:4294967295] 00000020:00000010:0.0:1784948690.357819:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000020:00000010:0.0:1784948690.357835:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000020:00000010:0.0:1784948690.357848:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008cc004d6. 00000400:00000200:1.0:1784948690.357849:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:0.0:1784948690.357862:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000800:00000200:1.0:1784948690.357869:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948690.357872:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000800:00000010:1.0:1784948690.357881:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000036e798e7. 00000400:00000200:3.0:1784948690.357882:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1784948690.357884:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948690.357893:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:3.0:1784948690.357895:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784948690.357898:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:3.0:1784948690.357944:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000800:00000200:1.0:1784948690.357951:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948690.357952:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784948690.357956:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948690.357966:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3125:3125:239:4294967295] 192.168.202.137@tcp LPNI seq info [2032:2032:7:4294967295] 00000800:00000200:1.0:1784948690.357976:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784948690.357978:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948690.357987:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000036e798e7 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:3.0:1784948690.358000:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948690.358004:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948690.358012:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948690.358016:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948690.358021:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000089a028d. 00000100:00000040:1.0:1784948690.358025:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:1.0:1784948690.358033:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305629740 00000100:00000001:0.0:1784948690.358040:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000200:3.0:1784948690.358041:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948690.358049:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305629739 00000100:00000001:0.0:1784948690.358051:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:3.0:1784948690.358062:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784948690.358066:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305629740 00000800:00000200:3.0:1784948690.358074:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000089a028d type 1, nob 320 niov 1 nkiov 1 00000100:00100000:0.0:1784948690.358076:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 682 00000100:00000001:1.0:1784948690.358078:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305629739 00000100:00000001:1.0:1784948690.358090:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305629740 00000100:00000040:0.0:1784948690.358096:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948690.358097:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305629739 00000100:00000001:3.0:1784948690.358101:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948690.358112:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466830928 : -115544242720688 : ffff96e9c274f050) 00000100:00000001:1.0:1784948690.358113:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305629740 00000100:00000040:3.0:1784948690.358114:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2017 usecs (3 jiffies) 00000100:00000001:1.0:1784948690.358127:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305629739 00000100:00000001:3.0:1784948690.358128:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305629701 00000100:00000040:0.0:1784948690.358138:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000008db39772 x1871651289646080/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:351/0 lens 224/0 e 0 to 0 dl 1784948701 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948690.358139:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305629740 00000100:00000001:1.0:1784948690.358141:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948690.358152:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305629701 00000100:00000001:1.0:1784948690.358154:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.358163:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305629740 00000100:00000001:1.0:1784948690.358166:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.358173:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305629701 00000100:00000001:1.0:1784948690.358178:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948690.358185:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948690.358186:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305629740 00000100:00000001:1.0:1784948690.358190:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784948690.358195:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:3.0:1784948690.358195:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305629701 00000100:00000001:1.0:1784948690.358196:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1784948690.358209:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305629740 00000100:00000001:0.0:1784948690.358210:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948690.358210:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948690.358217:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948690.358220:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948690.358222:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00000001:1.0:1784948690.358224:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.358227:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948690.358235:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.358237:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948690.358243:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000008db39772 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289646080:12345-0@lo:400: 00000100:00000001:3.0:1784948690.358247:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948690.358259:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289646080 00000100:00000001:3.0:1784948690.358262:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784948690.358266:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784948690.358268:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:0.0:1784948690.358275:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784948690.358277:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784948690.358282:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.358283:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:1.0:1784948690.358292:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948690.358293:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948690.358294:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948690.358302:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784948690.358305:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948690.358311:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948690.358317:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948690.358321:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948690.358328:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:1.0:1784948690.358333:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784948690.358334:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948690.358341:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948690.358349:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948690.358350:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948690.358358:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948690.358365:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:1.0:1784948690.358377:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000010:0.0:1784948690.358378:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000081e7de16. 02000000:00000001:0.0:1784948690.358387:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948690.358394:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948690.358404:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948690.358415:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6434480 00010000:00000040:0.0:1784948690.358416:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289646080 00010000:00000001:0.0:1784948690.358432:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000100:1.0:1784948690.358444:0:337114:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289646208 offset 0 length 520: 4 00010000:00000200:0.0:1784948690.358450:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000008db39772 x1871651289646080/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:351/0 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948690.358454:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1784948690.358458:0:337114:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 0000000075055a98 is in inconsistent state, don't perform health checking (-2, 0) 00000800:00000010:3.0:1784948690.358465:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000089a028d. 00000400:00000200:1.0:1784948690.358469:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:3.0:1784948690.358479:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948690.358489:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948690.358501:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948690.358507:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948690.358518:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000800:00000001:1.0:1784948690.358549:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:0.0:1784948690.358552:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000010:3.0:1784948690.358553:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000800:00000001:1.0:1784948690.358560:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948690.358563:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948690.358567:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948690.358568:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00001000:0.0:1784948690.358577:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=12 v=5 (1 1 1 1) 00000800:00000001:1.0:1784948690.358580:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948690.358582:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948706 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948690.358600:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784948690.358612:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:0.0:1784948690.358613:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948690.358626:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784948690.358628:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784948690.358636:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:3.0:1784948690.358638:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948706 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948690.358645:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948690.358646:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434500 00000100:00000001:0.0:1784948690.358659:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948690.358669:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.358679:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784948690.358681:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 148432 00000100:00000040:0.0:1784948690.358682:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948690.358694:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000800:00000001:1.0:1784948690.358701:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784948690.358706:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289646080, offset 224 00000400:00000200:0.0:1784948690.358721:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:1.0:1784948690.358727:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948690.358736:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948690.358740:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784948690.358751:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948690.358762:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434400 00000100:00000001:1.0:1784948690.358765:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948690.358773:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1784948690.358782:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7fc1 [1] + 224 00000100:00000010:1.0:1784948690.358787:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004f710a78. 00000100:00000040:1.0:1784948690.358796:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000004f710a78 x1871651289646336 msgsize 224 00000400:00000200:0.0:1784948690.358798:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948690.358815:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00100000:1.0:1784948690.358820:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948690.358827:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948690.358839:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948690.358857:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784948690.358863:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948690.358873:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948690.358886:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948690.358955:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948690.358972:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784948690.358975:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000100:00000001:3.0:1784948690.358979:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:0.0:1784948690.358986:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000100:00000001:0.0:1784948690.359000:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000001:3.0:1784948690.359002:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784948690.359010:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:3.0:1784948690.359012:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948690.359021:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a86c77e2. 00000100:00000001:0.0:1784948690.359025:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948690.359036:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000100:00000040:0.0:1784948690.359040:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000010:3.0:1784948690.359043:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00010000:00000001:0.0:1784948690.359055:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784948690.359057:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1784948690.359066:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948690.359072:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948690.359079:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948690.359097:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008db39772 x1871651289646080/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:351/0 lens 224/224 e 0 to 0 dl 1784948701 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948690.359180:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000008db39772 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289646080:12345-0@lo:400: Request processed in 990us (2033us total) trans 0 rc 0/0 00000100:00000200:3.0:1784948690.359182:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948690.359224:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 682 00000100:00000001:3.0:1784948690.359235:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1784948690.359236:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948690.359244:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948690.359251:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948690.359253:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784948690.359260:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:3.0:1784948690.359265:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:0.0:1784948690.359275:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008cc004d6. 00000100:00000001:3.0:1784948690.359275:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948690.359284:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948690.359291:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 02000000:00000001:3.0:1784948690.359293:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948690.359308:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000100:00001000:3.0:1784948690.359310:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=12 v=5 (5 5 5 5) 00000020:00000040:0.0:1784948690.359324:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:3.0:1784948690.359329:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=12 v=5 (1 1 1 1) 02000000:00000001:0.0:1784948690.359337:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948690.359346:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:0.0:1784948690.359347:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000081e7de16. 00000100:00000001:3.0:1784948690.359354:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948690.359362:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:3.0:1784948690.359367:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784948690.359372:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784948690.359374:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.359390:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948690.359391:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:0.0:1784948690.359410:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000968c2fa4. 00000100:00000040:3.0:1784948690.359411:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948690.359427:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000feb5569e. 00000100:00000001:3.0:1784948690.359436:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948690.359444:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948690.359450:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948690.359458:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948690.359475:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000035b51214 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289646080:0@lo:400: 00000100:00000001:3.0:1784948690.359490:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948690.359496:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948690.359505:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000035b51214 x1871651289646080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948690.359568:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948690.359576:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948690.359584:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000f1904ca. 02000000:00000001:3.0:1784948690.359595:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948690.359601:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948690.359609:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784948690.359616:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948690.359622:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000004805f207. 02000000:00000001:3.0:1784948690.359629:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948690.359634:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948690.359642:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000035b51214. 00000100:00000001:3.0:1784948690.359651:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948690.359657:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948690.359661:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948690.359672:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948690.359679:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948690.359685:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948690.359688:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948690.359967:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784948690.359973:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784948690.359977:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009bb74647. 00000800:00000001:3.0:1784948690.359986:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948690.359989:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948690.359992:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948690.360001:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948690.360010:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948690.360021:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948690.360027:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:0.0:1784948690.360035:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:3.0:1784948690.360037:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434500 00000100:00000001:0.0:1784948690.360047:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:3.0:1784948690.360053:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x7fcd [1] + 224 00000100:00000001:0.0:1784948690.360056:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948690.360064:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948690.360076:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948690.360079:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948690.360085:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948690.360093:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948690.360098:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948690.360110:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fa1e143a x1871651289646336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948690.360133:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948690.360141:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948690.360144:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948690.401029:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305629740, at 4305629741 00000100:00000040:3.0:1784948690.401034:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305629740, at 4305629741 00000100:00000010:0.0:1784948690.401055:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f68fd7be. 00000100:00000040:1.0:1784948690.401058:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305629740, at 4305629741 00000100:00000010:3.0:1784948690.401059:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000abbd7d53. 00000100:00000010:1.0:1784948690.401077:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000824e9543. 00000020:00000001:1.0:1784948691.188198:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948691.188205:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948691.188208:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000010:3.0:1784948691.572034:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001e8db685. 00000400:00000010:3.0:1784948691.572062:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000011546f5a. 00000400:00000010:3.0:1784948691.572078:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000249338ff. 00000400:00000010:3.0:1784948691.572083:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000077b9b581. 00000400:00000010:3.0:1784948691.572087:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000003ac0766. 00000400:00000010:3.0:1784948691.572092:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000079f1d10e. 00000400:00000010:3.0:1784948691.572095:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b7bd3e11. 00000400:00000010:3.0:1784948691.572099:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c0a3f621. 00000400:00000010:3.0:1784948691.572102:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007174c252. 00000400:00000010:3.0:1784948691.572106:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000074570d70. 00000400:00000010:3.0:1784948691.572109:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000534ace9d. 00000400:00000010:3.0:1784948691.572113:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f99d1b9f. 00000400:00000010:3.0:1784948691.572117:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009b1df9a1. 00000400:00000010:3.0:1784948691.572121:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a400fa42. 00000400:00000010:3.0:1784948691.572125:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000039ee8127. 00000400:00000010:3.0:1784948691.572129:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000017f0d388. 00000400:00000010:3.0:1784948691.572133:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b666dac0. 00000400:00000010:3.0:1784948691.572137:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ae8b415c. 00000400:00000010:3.0:1784948691.572140:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000905e873. 00000400:00000010:3.0:1784948691.572144:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000399be696. 00000400:00000010:3.0:1784948691.572148:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002b26853b. 00000400:00000010:3.0:1784948691.572152:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cc444ae0. 00000400:00000010:3.0:1784948691.572156:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007701057d. 00000400:00000010:3.0:1784948691.572160:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e8caef65. 00000400:00000010:3.0:1784948691.572164:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000097c19a4c. 00000400:00000010:3.0:1784948691.572168:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b64f85b3. 00000400:00000010:3.0:1784948691.572173:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006bdf3597. 00000400:00000010:3.0:1784948691.572181:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000147b89d3. 00000400:00000010:3.0:1784948691.572187:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003e18501d. 00000400:00000010:3.0:1784948691.572190:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000038468d93. 00000400:00000010:3.0:1784948691.572193:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008c245eeb. 00000400:00000010:3.0:1784948691.572195:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cf871ac3. 00000400:00000010:3.0:1784948691.572198:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008015f6b2. 00000400:00000010:3.0:1784948691.572200:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008b69f56d. 00000400:00000010:3.0:1784948691.572204:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000003b6cd72. 00000400:00000010:3.0:1784948691.572207:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000600a4ae0. 00000400:00000010:3.0:1784948691.572210:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006091982f. 00000400:00000010:3.0:1784948691.572212:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e2f3909c. 00000400:00000010:3.0:1784948691.572215:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001e82affc. 00000400:00000010:3.0:1784948691.572217:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000034614846. 00000400:00000010:3.0:1784948691.572220:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000012554f3a. 00000400:00000010:3.0:1784948691.572222:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009f5e55af. 00000400:00000010:3.0:1784948691.572224:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c732a1a8. 00000400:00000010:3.0:1784948691.572227:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000390dd32d. 00000400:00000010:3.0:1784948691.572229:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e42cbe57. 00000400:00000010:3.0:1784948691.572231:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c12c1931. 00000400:00000010:3.0:1784948691.572233:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002e599554. 00000400:00000010:3.0:1784948691.572236:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000fd8e815. 00000400:00000010:3.0:1784948691.572238:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f9462ca3. 00000400:00000010:3.0:1784948691.572240:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000087a70a9. 00000400:00000010:3.0:1784948691.572242:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000067f76b46. 00000400:00000010:3.0:1784948691.572245:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000024644ba3. 00000400:00000010:3.0:1784948691.572247:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a2b37178. 00000400:00000010:3.0:1784948691.572249:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d5bb2c49. 00000400:00000010:3.0:1784948691.572251:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e69b1edb. 00000400:00000010:3.0:1784948691.572253:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000b1d42ed. 00000400:00000010:3.0:1784948691.572256:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000098ebc0ae. 00000400:00000010:3.0:1784948691.572258:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bfff640a. 00000400:00000010:3.0:1784948691.572260:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004d156103. 00000400:00000010:3.0:1784948691.572262:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000fee11282. 00000400:00000010:3.0:1784948691.572265:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000005c4dcc18. 00000400:00000010:3.0:1784948691.572267:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a0e11faa. 00000400:00000010:3.0:1784948691.572269:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001bae6561. 00000400:00000010:3.0:1784948691.572271:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a705132d. 00000400:00000010:3.0:1784948691.572274:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c241e266. 00000400:00000010:3.0:1784948691.572276:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000548ea7b. 00000400:00000010:3.0:1784948691.572279:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003859b1c5. 00000400:00000010:3.0:1784948691.572281:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000afa63392. 00000400:00000010:3.0:1784948691.572283:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003faefc6b. 00000400:00000010:3.0:1784948691.572285:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f12dd04d. 00000020:00000001:3.0:1784948692.212224:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948692.212230:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948692.212234:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948693.236226:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948693.236237:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948693.236242:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948694.260247:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948694.260253:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948694.260257:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948695.284083:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948695.284088:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948695.284091:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948695.475996:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948695.476009:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948695.476017:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b47dc450. 00000100:00000001:0.0:1784948695.476024:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000020:00000040:1.0:1784948695.476025:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948695.476032:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948695.476035:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1784948695.476037:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784948695.476038:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948695.476042:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948695.476045:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784948695.476045:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948695.476049:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948695.476052:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948695.476056:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003ed79484. 00000100:00000001:1.0:1784948695.476065:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.476075:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948695.476097:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948695.476100:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.476105:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b47dc450] to pc [ptlrpcd_00_03+3] req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.476132:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948695.476135:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948695.476139:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948695.476143:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948695.476148:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948695.476154:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784948695.476154:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b93d109. 00000100:00000001:0.0:1784948695.476159:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948695.476160:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948695.476166:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948695.476169:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948695.476174:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948695.476181:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:1.0:1784948695.476187:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.476198:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948695.476219:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948695.476223:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.476228:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005b93d109] to pc [ptlrpcd_00_00+0] req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.476258:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.476265:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10972) 00000100:00000001:1.0:1784948695.476279:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948695.476284:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948695.476284:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948695.476288:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948695.476290:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.476296:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.476310:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948695.476313:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948695.476316:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948695.476320:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948695.476326:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948695.476333:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289646592:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948695.476339:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948695.476341:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948695.476344:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948695.476349:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948695.476356:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000002c1a3d41. 02000000:00000001:1.0:1784948695.476359:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948695.476366:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eadc14f. 00000400:00000010:1.0:1784948695.476384:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000200:1.0:1784948695.476392:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289646592, portal 10 00000100:00000001:1.0:1784948695.476398:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948695.476400:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451647872 : -115544257903744 : ffff96e9c18d4380) 00000100:00000040:1.0:1784948695.476413:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.476436:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948695.476440:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948695.476443:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00000200:1.0:1784948695.476447:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289646592, offset 0 00000400:00000200:1.0:1784948695.476454:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948695.476466:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948695.476475:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948695.476500:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948695.476509:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948695.476516:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948695.476522:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3127:3127:240:4294967295] 192.168.202.137@tcp LPNI seq info [2034:2034:8:4294967295] 00000100:00000001:3.0:1784948695.476526:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948695.476530:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289646464 02000000:00000001:3.0:1784948695.476534:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948695.476537:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948695.476540:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948695.476541:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:3.0:1784948695.476546:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948695.476550:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289646464 00000800:00000200:1.0:1784948695.476553:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784948695.476554:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948695.476558:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784948695.476561:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000010:1.0:1784948695.476562:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000662152ce. 00000020:00000040:3.0:1784948695.476565:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:3.0:1784948695.476570:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:1.0:1784948695.476575:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:3.0:1784948695.476575:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948695.476582:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948695.476588:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000cb7271f. 00000800:00000200:1.0:1784948695.476591:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784948695.476595:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1784948695.476597:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000662152ce type 1, nob 320 niov 1 nkiov 1 00000020:00000010:3.0:1784948695.476601:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000cb7271f. 00000020:00000010:3.0:1784948695.476610:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000100:00000001:1.0:1784948695.476614:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948695.476618:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dc76901d. 00000100:00000040:1.0:1784948695.476620:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2020 usecs (3 jiffies) 00000020:00000010:3.0:1784948695.476624:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000e923a07a. 00000100:00000001:1.0:1784948695.476628:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305634819 00000100:00000040:3.0:1784948695.476633:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948695.476635:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305634858 00000100:00000001:3.0:1784948695.476637:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948695.476641:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948695.476642:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305634819 00000100:00000001:3.0:1784948695.476646:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.476657:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305634858 00000100:00000001:3.0:1784948695.476657:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.476664:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305634819 00000100:00000001:1.0:1784948695.476669:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305634858 00000100:00000001:1.0:1784948695.476675:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305634819 00000100:00000001:3.0:1784948695.476676:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948695.476681:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305634858 00000100:00000001:1.0:1784948695.476685:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948695.476688:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.476692:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.476696:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948695.476701:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948695.476703:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948695.476705:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948695.476706:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948695.476707:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948695.476711:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948695.476712:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 683 00000100:00000001:3.0:1784948695.476713:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948695.476715:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948695.476718:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948695.476719:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948695.476719:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948695.476724:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466827856 : -115544242723760 : ffff96e9c274e450) 00000100:00000001:3.0:1784948695.476725:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.476726:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948695.476733:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000875035e8 x1871651289646464/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:356/0 lens 224/0 e 0 to 0 dl 1784948706 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948695.476740:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948695.476743:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000043fe70d5. 00000100:00000001:0.0:1784948695.476745:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948695.476747:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000043fe70d5. 00000100:00100000:0.0:1784948695.476751:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000875035e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289646464:12345-0@lo:400: 00000100:00000200:0.0:1784948695.476754:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289646464 00000020:00000001:0.0:1784948695.476758:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948695.476761:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948695.476763:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948695.476766:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948695.476767:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948695.476770:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948695.476771:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948695.476774:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948695.476776:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948695.476778:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948695.476780:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948695.476782:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948695.476783:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948695.476787:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000081e7de16. 02000000:00000001:0.0:1784948695.476788:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948695.476790:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948695.476791:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948695.476793:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289646464 00010000:00000001:0.0:1784948695.476795:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948695.476798:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000875035e8 x1871651289646464/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:356/0 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948695.476804:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948695.476805:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948695.476808:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=17 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948695.476811:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948695.476813:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948695.476815:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948695.476817:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948695.476819:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948695.476821:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948695.476822:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948695.476824:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000100:00000200:0.0:1784948695.476827:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289646464, offset 224 00000400:00000200:0.0:1784948695.476831:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948695.476837:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948695.476843:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434580 00000400:00000200:0.0:1784948695.476847:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x7fe5 [1] + 224 00000400:00000200:0.0:1784948695.476851:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948695.476854:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948695.476857:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948695.476859:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948695.476867:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784948695.476871:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948695.476879:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000662152ce. 00000100:00000001:0.0:1784948695.476885:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948695.476886:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948695.476891:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000400:00000010:0.0:1784948695.476895:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000400:00000200:1.0:1784948695.476898:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948695.476902:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:1.0:1784948695.476929:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000100:00000001:0.0:1784948695.476931:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000010:1.0:1784948695.476935:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000100:00000001:0.0:1784948695.476938:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.476943:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784948695.476946:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784948695.476951:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:1.0:1784948695.476952:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948695.476954:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948695.476959:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948695.476967:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000875035e8 x1871651289646464/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:356/0 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.476969:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948695.476976:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948695.476981:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948695.476983:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000875035e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289646464:12345-0@lo:400: Request processed in 238us (563us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948695.476985:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948695.476992:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948695.476997:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948695.477000:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005cc7db3b. 00000100:00100000:0.0:1784948695.477001:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 683 00000100:00000001:1.0:1784948695.477006:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948695.477007:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:3.0:1784948695.477008:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000100:00000001:1.0:1784948695.477010:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948695.477013:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:3.0:1784948695.477013:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000100:00000001:0.0:1784948695.477017:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948695.477018:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784948695.477022:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dc76901d. 00000100:00000200:3.0:1784948695.477025:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948695.477028:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000e923a07a. 00000020:00000010:0.0:1784948695.477035:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000020:00000040:0.0:1784948695.477044:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:3.0:1784948695.477044:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948695.477050:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784948695.477053:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000081e7de16. 02000000:00000001:0.0:1784948695.477059:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948695.477061:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948695.477061:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948695.477064:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948695.477066:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784948695.477068:0:337122:0:(client.c:1580:after_reply()) Process entered 00000400:00000010:0.0:1784948695.477072:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000093aab5da. 02000000:00000001:3.0:1784948695.477072:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948695.477075:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:0.0:1784948695.477077:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009412aeee. 00000100:00000001:3.0:1784948695.477079:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948695.477083:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948695.477087:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=17 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948695.477092:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=17 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948695.477096:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948695.477098:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.477102:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:3.0:1784948695.477102:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784948695.477104:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948695.477106:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289646592 00000100:00000001:3.0:1784948695.477109:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948695.477110:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948695.477112:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948695.477115:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948695.477118:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948695.477120:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948695.477125:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289646592 00000020:00000001:1.0:1784948695.477129:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948695.477132:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948695.477136:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948695.477137:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784948695.477140:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:3.0:1784948695.477142:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1784948695.477144:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948695.477145:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948695.477150:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:3.0:1784948695.477151:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.477156:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948695.477162:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000030655a79. 00000100:00000001:1.0:1784948695.477166:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948695.477171:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000030655a79. 00000100:00100000:3.0:1784948695.477177:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b47dc450 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289646464:0@lo:400: 00000020:00000010:1.0:1784948695.477179:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007465fe15. 00000020:00000010:1.0:1784948695.477186:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cf558f26. 00000100:00000001:3.0:1784948695.477188:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948695.477192:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1784948695.477193:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:3.0:1784948695.477196:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b47dc450 x1871651289646464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948695.477203:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948695.477207:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1784948695.477208:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948695.477211:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784948695.477212:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1784948695.477214:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784948695.477215:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:3.0:1784948695.477220:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1784948695.477223:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000020:00000001:3.0:1784948695.477223:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948695.477224:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948695.477228:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000001:3.0:1784948695.477228:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:1.0:1784948695.477231:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948695.477232:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003ed79484. 02000000:00000001:3.0:1784948695.477236:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948695.477237:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948695.477239:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b47dc450. 00000100:00000001:3.0:1784948695.477241:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948695.477242:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948695.477244:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948695.477244:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948695.477252:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948695.477254:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948695.477257:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948695.477258:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948695.477259:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.477260:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948695.477267:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 659 00000100:00000040:1.0:1784948695.477269:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948695.477271:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470013440 : -115544239538176 : ffff96e9c2a58000) 00000100:00000040:1.0:1784948695.477275:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000c2ff6aed x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:356/0 lens 224/0 e 0 to 0 dl 1784948706 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.477281:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948695.477283:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000030655a79. 00000100:00000001:1.0:1784948695.477285:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948695.477287:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000030655a79. 00000100:00100000:1.0:1784948695.477290:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000c2ff6aed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289646592:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948695.477292:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289646592 00000020:00000001:1.0:1784948695.477294:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948695.477296:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948695.477297:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948695.477298:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948695.477299:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948695.477301:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948695.477303:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948695.477304:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948695.477305:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948695.477307:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948695.477308:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948695.477311:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948695.477312:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948695.477314:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000007cdbbc4. 02000000:00000001:1.0:1784948695.477315:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.477316:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948695.477318:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948695.477319:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289646592 00010000:00000001:1.0:1784948695.477320:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948695.477322:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000c2ff6aed x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:356/0 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948695.477327:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948695.477328:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948695.477330:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=12 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948695.477332:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948695.477333:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948695.477334:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948695.477335:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948695.477336:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.477338:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948695.477339:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948695.477341:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00000200:1.0:1784948695.477343:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289646592, offset 224 00000400:00000200:1.0:1784948695.477345:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948695.477350:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948695.477354:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3128:3128:240:4294967295] 192.168.202.137@tcp LPNI seq info [2035:2035:8:4294967295] 00000400:00000200:1.0:1784948695.477362:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948695.477367:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948695.477370:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000662152ce. 00000800:00000200:1.0:1784948695.477375:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948695.477381:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948695.477383:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000662152ce type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948695.477395:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.477398:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948695.477401:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948695.477403:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948695.477405:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948695.477409:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c2ff6aed x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:356/0 lens 224/224 e 0 to 0 dl 1784948706 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948695.477418:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000c2ff6aed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289646592:12345-192.168.202.137@tcp:400: Request processed in 136us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948695.477426:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 659 00000100:00000040:1.0:1784948695.477429:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948695.477432:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948695.477434:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948695.477436:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cf558f26. 00000020:00000010:1.0:1784948695.477440:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948695.477443:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007465fe15. 00000020:00000040:1.0:1784948695.477448:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948695.477450:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948695.477462:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948695.477465:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000662152ce. 00000400:00000200:3.0:1784948695.477469:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948695.477474:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948695.477487:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:3.0:1784948695.477489:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000100:00000001:3.0:1784948695.477493:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948695.477494:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948695.477578:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948695.477581:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948695.477584:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948695.477585:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948695.477586:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002eadc14f. 00000400:00000200:1.0:1784948695.477589:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:1.0:1784948695.477590:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000100:00000001:1.0:1784948695.477592:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948695.477595:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948695.477601:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.477606:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948695.477607:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948695.477608:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948695.477609:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948695.477610:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948695.477611:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948695.477612:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948695.477615:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=12 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948695.477618:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=12 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948695.477619:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948695.477620:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948695.477621:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948695.477623:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948695.477628:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.477630:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948695.477631:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948695.477632:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948695.477632:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948695.477634:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.477638:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948695.477640:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948695.477641:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948695.477643:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948695.477647:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289646592:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948695.477649:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948695.477650:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948695.477652:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005b93d109 x1871651289646592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948695.477659:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948695.477661:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948695.477662:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000002c1a3d41. 02000000:00000001:1.0:1784948695.477663:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948695.477664:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948695.477665:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948695.477666:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948695.477667:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:1.0:1784948695.477669:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948695.477670:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948695.477671:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b93d109. 00000100:00000001:1.0:1784948695.477672:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948695.477673:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948695.477674:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948695.477677:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948695.477678:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948695.477679:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948695.477680:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948695.518969:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305634858, at 4305634859 00000100:00000040:1.0:1784948695.518969:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305634858, at 4305634859 00000100:00000040:3.0:1784948695.518969:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305634858, at 4305634859 00000100:00000010:0.0:1784948695.518986:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f4af3cce. 00000100:00000010:1.0:1784948695.518990:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000010:3.0:1784948695.518992:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a3d0cc2f. 00000020:00000001:3.0:1784948696.308101:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948696.308105:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948696.308108:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948696.499972:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948696.499992:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948696.499994:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948696.499999:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948696.500002:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948696.500005:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948696.500007:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948697.332121:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948697.332126:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948697.332129:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948697.524042:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948697.524060:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948697.524063:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948697.524069:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948697.524072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948697.524076:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948697.524080:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948698.356182:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948698.356188:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948698.356192:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948698.548021:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948698.548040:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948698.548058:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948698.548064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948698.548067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948698.548071:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948698.548075:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948699.380108:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948699.380111:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948699.380113:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948699.572001:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948699.572014:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948699.572017:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948699.572021:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948699.572024:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948699.572028:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948699.572031:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948700.404167:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948700.404172:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948700.404176:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948700.596034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1784948700.596046:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948700.596056:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948700.596057:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948700.596065:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784948700.596070:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b93d109. 00000100:00000001:0.0:1784948700.596074:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948700.596081:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948700.596084:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948700.596093:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948700.596094:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948700.596099:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784948700.596103:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948700.596108:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948700.596120:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:1.0:1784948700.596129:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948700.596141:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948700.596156:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948700.596160:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948700.596167:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005b93d109] to pc [ptlrpcd_00_01+1] req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948700.596225:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948700.596229:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948700.596240:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948700.596245:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948700.596250:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948700.596255:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a07b5f33. 00000020:00000040:1.0:1784948700.596260:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948700.596264:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948700.596267:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948700.596269:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948700.596275:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002a753d9d. 00000100:00000001:0.0:1784948700.596279:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948700.596281:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.596283:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948700.596290:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948700.596293:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948700.596295:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948700.596297:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784948700.596310:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948700.596320:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948700.596327:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948700.596345:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784948700.596347:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a07b5f33] to pc [ptlrpcd_00_02+2] req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948700.596353:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948700.596367:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948700.596374:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948700.596383:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948700.596391:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289646720:0@lo:400: 00000100:00000001:0.0:1784948700.596402:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784948700.596407:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948700.596412:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948700.596420:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948700.596423:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10977) 02000000:00000001:0.0:1784948700.596427:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948700.596458:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948700.596466:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948700.596479:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000055244775. 00000100:00000001:3.0:1784948700.596487:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784948700.596492:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000100:00000001:1.0:1784948700.596493:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784948700.596495:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:1.0:1784948700.596504:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.596504:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948700.596506:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289646720, portal 25 00000100:00000001:0.0:1784948700.596517:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948700.596517:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948700.596522:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451647872 : -115544257903744 : ffff96e9c18d4380) 00000100:00000001:3.0:1784948700.596522:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.596528:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948700.596531:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948700.596534:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948700.596535:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948700.596545:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948700.596550:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948700.596557:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784948700.596561:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:0.0:1784948700.596564:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000001:3.0:1784948700.596564:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948700.596569:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289646720, offset 0 02000000:00000001:3.0:1784948700.596569:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948700.596573:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948700.596578:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948700.596582:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:3.0:1784948700.596584:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a07b5f33 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289646848:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948700.596591:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948700.596596:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948700.596599:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948700.596602:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948700.596603:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:3.0:1784948700.596609:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005e186afc. 02000000:00000001:3.0:1784948700.596613:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948700.596622:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434680 00000400:00000010:3.0:1784948700.596622:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000400:00000010:3.0:1784948700.596631:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000400:00000200:0.0:1784948700.596636:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x75e1 [2] + 0 00000400:00000010:0.0:1784948700.596644:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ea556a29. 00000400:00000200:0.0:1784948700.596655:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000004dd24a2e 00000100:00000200:3.0:1784948700.596661:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289646848, portal 10 00000100:00000001:3.0:1784948700.596666:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1784948700.596667:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948700.596670:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451655936 : -115544257895680 : ffff96e9c18d6300) 00000400:00000200:0.0:1784948700.596676:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:3.0:1784948700.596683:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948700.596688:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004dd24a2e 00000400:00000010:0.0:1784948700.596693:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000004dd24a2e. 00000100:00000001:3.0:1784948700.596697:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948700.596703:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784948700.596703:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:0.0:1784948700.596708:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:3.0:1784948700.596710:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000100:00000040:0.0:1784948700.596714:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000019a655f1 x1871651289646720 msgsize 224 00000100:00000200:3.0:1784948700.596716:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289646848, offset 0 00000100:00100000:0.0:1784948700.596724:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784948700.596725:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784948700.596729:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948700.596740:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948700.596750:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784948700.596753:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948700.596757:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948700.596763:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948700.596769:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948700.596777:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3129:3129:240:4294967295] 192.168.202.137@tcp LPNI seq info [2036:2036:8:4294967295] 00000400:00000200:3.0:1784948700.596806:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948700.596817:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948700.596820:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784948700.596826:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000800:00000010:3.0:1784948700.596831:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004805f207. 00000100:00000001:0.0:1784948700.596838:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784948700.596847:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784948700.596852:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948700.596867:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948700.596878:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004805f207 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948700.596884:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948700.596893:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948700.596948:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948700.596959:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2018 usecs (3 jiffies) 00000100:00000001:0.0:1784948700.596964:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.596969:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305639940 00000100:00000001:0.0:1784948700.596973:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:3.0:1784948700.596981:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:0.0:1784948700.596986:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948700.596992:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305639979 00000100:00000001:3.0:1784948700.597043:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948700.597078:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305639940 00000100:00000010:3.0:1784948700.597089:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000afdabce. 00000100:00000001:3.0:1784948700.597098:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948700.597109:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305639940 00000100:00000010:3.0:1784948700.597121:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:3.0:1784948700.597134:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948700.597151:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305639940 00000100:00000010:3.0:1784948700.597167:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000800:00000200:0.0:1784948700.597177:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948700.597178:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000800:00000010:0.0:1784948700.597186:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004805f207. 00000100:00000001:3.0:1784948700.597187:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948700.597198:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948700.597201:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948700.597208:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.597213:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948700.597218:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948700.597225:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000400:00000010:0.0:1784948700.597229:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000100:00000001:0.0:1784948700.597236:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:3.0:1784948700.597239:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784948700.597244:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948700.597256:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948700.597260:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:3.0:1784948700.597260:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784948700.597268:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948700.597269:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948700.597282:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948700.597283:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948700.597287:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784948700.597317:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434700 00000100:00000001:0.0:1784948700.597324:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305639940 00000400:00000200:3.0:1784948700.597335:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 148880 00000100:00000001:0.0:1784948700.597339:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305639979 00000100:00000001:0.0:1784948700.597355:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305639940 00000800:00000001:3.0:1784948700.597355:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948700.597372:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305639979 00000800:00000001:3.0:1784948700.597372:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948700.597384:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948700.597387:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305639940 00000100:00000001:0.0:1784948700.597397:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305639979 00000400:00000200:3.0:1784948700.597401:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948700.597408:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305639940 00000100:00000001:3.0:1784948700.597416:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784948700.597421:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784948700.597421:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784948700.597429:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948700.597431:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000035b51214. 00000100:00000001:0.0:1784948700.597468:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948700.597468:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000035b51214 x1871651289646848 msgsize 224 00000100:00000001:0.0:1784948700.597480:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948700.597493:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948700.597495:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948700.597501:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948700.597509:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948700.597513:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948700.597520:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:3.0:1784948700.597523:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948700.597528:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948700.597532:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948700.597538:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948700.597545:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948700.597554:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289646848 02000000:00000001:1.0:1784948700.597563:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948700.597569:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948700.597574:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948700.597584:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948700.597591:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289646848 00000020:00000001:1.0:1784948700.597599:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948700.597604:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948700.597609:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948700.597614:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948700.597622:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948700.597629:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948700.597636:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948700.597644:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784948700.597690:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948700.597695:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000020:00000010:1.0:1784948700.597702:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007465fe15. 00000020:00000010:1.0:1784948700.597707:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cf558f26. 00000020:00000010:1.0:1784948700.597713:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:1.0:1784948700.597724:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948700.597731:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948700.597734:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948700.597739:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948700.597750:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948700.597756:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948700.597763:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948700.597782:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948700.597803:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948700.597808:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948700.597820:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 660 00000100:00000040:1.0:1784948700.597825:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948700.597829:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482515328 : -115544227036288 : ffff96e9c3644380) 00000100:00000040:1.0:1784948700.597843:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000035b51214 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:361/0 lens 224/0 e 0 to 0 dl 1784948711 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948700.597864:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948700.597874:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948700.597887:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784948700.597891:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000001:3.0:1784948700.597893:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948700.597902:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000055244775. 00000020:00000010:1.0:1784948700.597947:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784948700.597961:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948700.597964:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000020:00000010:1.0:1784948700.597975:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000400:00000010:3.0:1784948700.597975:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000100:00000001:3.0:1784948700.597992:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784948700.598003:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000035b51214 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289646848:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784948700.598010:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948700.598020:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289646848 00000020:00000001:1.0:1784948700.598029:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948700.598039:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948700.598046:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948700.598054:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948700.598058:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000200:3.0:1784948700.598062:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948700.598069:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948700.598076:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948700.598086:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948700.598093:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784948700.598097:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1784948700.598098:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.598104:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948700.598107:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948700.598114:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:1.0:1784948700.598117:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948700.598120:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:1.0:1784948700.598125:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784948700.598126:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948700.598133:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948700.598134:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000885be19. 02000000:00000001:3.0:1784948700.598140:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948700.598143:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948700.598151:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948700.598152:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=22 v=5 (5 5 5 5) 00000020:00000001:1.0:1784948700.598160:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948700.598163:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=22 v=5 (1 1 1 1) 00010000:00000040:1.0:1784948700.598167:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289646848 00000100:00000001:3.0:1784948700.598170:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00010000:00000001:1.0:1784948700.598173:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948700.598175:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948700.598180:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000200:1.0:1784948700.598186:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000035b51214 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:361/0 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948700.598187:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.598197:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948700.598209:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948700.598214:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948700.598220:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948700.598229:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=17 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948700.598230:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948700.598242:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948700.598244:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784948700.598246:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948700.598254:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000040:3.0:1784948700.598259:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948700.598263:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948700.598275:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948700.598279:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948700.598288:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1784948700.598289:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289646720:0@lo:400: 00000100:00000040:1.0:1784948700.598299:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948700.598305:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948700.598307:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000001:3.0:1784948700.598311:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1784948700.598317:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289646848, offset 224 00000100:00000040:3.0:1784948700.598318:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005b93d109 x1871651289646720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948700.598334:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948700.598341:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:1.0:1784948700.598342:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000010:3.0:1784948700.598347:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 02000000:00000001:3.0:1784948700.598356:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948700.598359:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948700.598363:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:1.0:1784948700.598368:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000020:00000001:3.0:1784948700.598369:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948700.598375:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:3.0:1784948700.598384:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948700.598389:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948700.598393:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b93d109. 00000400:00000200:1.0:1784948700.598395:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3130:3130:240:4294967295] 192.168.202.137@tcp LPNI seq info [2037:2037:8:4294967295] 00000100:00000001:3.0:1784948700.598399:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948700.598404:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.598409:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.598418:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948700.598423:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.598430:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948700.598465:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948700.598469:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948700.598480:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948700.598486:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000800:00000200:1.0:1784948700.598494:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948700.598501:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948700.598505:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948700.598523:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948700.598527:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948700.598530:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948700.598532:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948700.598535:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948700.598540:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000035b51214 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:361/0 lens 224/224 e 0 to 0 dl 1784948711 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948700.598553:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000035b51214 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289646848:12345-192.168.202.137@tcp:400: Request processed in 661us (1084us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948700.598564:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 660 00000100:00000040:1.0:1784948700.598568:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948700.598570:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948700.598574:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948700.598580:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cf558f26. 00000020:00000010:1.0:1784948700.598585:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948700.598589:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007465fe15. 00000020:00000040:1.0:1784948700.598596:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948700.598599:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948700.598711:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948700.598725:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948700.598731:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948700.598735:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948700.598747:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948700.598759:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434700 00000400:00000200:0.0:1784948700.598766:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8009 [1] + 224 00000800:00000001:0.0:1784948700.598772:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948700.598780:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948700.598783:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948700.598787:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948700.598793:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948700.598798:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948700.598809:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948700.598833:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948700.598841:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948700.598846:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.598892:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948700.598897:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948700.598930:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948700.598933:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948700.598936:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000400:00000200:3.0:1784948700.598941:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000400:00000010:3.0:1784948700.598944:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000100:00000001:3.0:1784948700.598947:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948700.598952:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948700.598963:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948700.598971:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948700.598973:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.598975:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948700.598978:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948700.598980:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948700.598982:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948700.598984:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948700.598990:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=17 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948700.598993:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=17 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948700.598997:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948700.598998:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948700.599001:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948700.599004:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948700.599013:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.599017:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948700.599019:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948700.599021:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948700.599022:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948700.599025:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948700.599033:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948700.599036:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948700.599038:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948700.599041:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948700.599049:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a07b5f33 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289646848:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948700.599054:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.599056:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948700.599058:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a07b5f33 x1871651289646848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948700.599066:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948700.599067:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948700.599069:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000005e186afc. 02000000:00000001:3.0:1784948700.599074:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948700.599076:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948700.599078:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948700.599080:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948700.599082:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002a753d9d. 02000000:00000001:3.0:1784948700.599084:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948700.599086:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948700.599088:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a07b5f33. 00000100:00000001:3.0:1784948700.599091:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948700.599093:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.599095:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948700.599099:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948700.599101:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948700.599103:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948700.599105:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948700.640010:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305639979, at 4305639980 00000100:00000040:0.0:1784948700.640019:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305639979, at 4305639980 00000100:00000010:3.0:1784948700.640029:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:0.0:1784948700.640036:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00000100:00000040:1.0:1784948700.640050:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305639979, at 4305639980 00000100:00000010:1.0:1784948700.640069:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000afdabce. 00000020:00000001:1.0:1784948701.428076:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948701.428080:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948701.428083:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948701.620039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948701.620058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948701.620061:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948701.620068:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948701.620072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948701.620077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948701.620080:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948702.452230:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948702.452237:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948702.452244:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948702.644007:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948702.644022:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948702.644024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948702.644030:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948702.644034:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948702.644037:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948702.644040:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948703.476138:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948703.476142:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948703.476145:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948704.500120:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948704.500127:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948704.500131:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948705.524178:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948705.524185:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948705.524189:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948705.716034:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948705.716051:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948705.716063:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a8233e67. 00000020:00000040:1.0:1784948705.716074:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948705.716083:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948705.716089:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948705.716098:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948705.716112:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000008f5595bf. 00000100:00000001:1.0:1784948705.716124:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948705.716139:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948705.716160:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948705.716164:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948705.716172:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8233e67] to pc [ptlrpcd_00_03+3] req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948705.716228:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948705.716234:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948705.716242:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948705.716247:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948705.716253:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948705.716259:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b55e20f2. 00000020:00000040:1.0:1784948705.716287:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948705.716291:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948705.716295:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948705.716299:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948705.716300:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1784948705.716305:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00100000:0.0:1784948705.716307:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:1.0:1784948705.716310:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948705.716311:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948705.716320:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948705.716322:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948705.716326:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948705.716330:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948705.716350:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948705.716353:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948705.716372:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b55e20f2] to pc [ptlrpcd_00_00+0] req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948705.716373:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948705.716377:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948705.716386:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948705.716451:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948705.716457:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.716460:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948705.716462:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948705.716467:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948705.716469:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10982) 02000000:00000001:0.0:1784948705.716472:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948705.716476:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a8233e67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289646976:0@lo:400: 00000100:00000001:0.0:1784948705.716485:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948705.716493:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948705.716496:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948705.716498:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948705.716505:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948705.716507:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948705.716517:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784948705.716518:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948705.716524:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948705.716524:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1784948705.716533:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948705.716537:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c7491352. 00000100:00000001:1.0:1784948705.716549:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:0.0:1784948705.716551:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000001:1.0:1784948705.716556:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.716561:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784948705.716564:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289646976, portal 25 00000100:00000001:1.0:1784948705.716566:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948705.716572:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948705.716572:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948705.716578:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451649664 : -115544257901952 : ffff96e9c18d4a80) 00000100:00000040:1.0:1784948705.716583:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948705.716588:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948705.716599:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948705.716601:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948705.716601:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948705.716606:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948705.716607:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784948705.716611:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948705.716613:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 02000000:00000001:1.0:1784948705.716618:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948705.716620:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289646976, offset 0 00000100:00100000:1.0:1784948705.716629:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289647104:192.168.202.137@tcp:400: 00000400:00000200:0.0:1784948705.716633:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948705.716636:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948705.716641:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948705.716644:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948705.716648:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948705.716654:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948705.716674:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434780 00000400:00000200:0.0:1784948705.716685:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7601 [2] + 0 00000400:00000010:0.0:1784948705.716692:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000aba79327. 00000400:00000200:0.0:1784948705.716700:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f0339559 00000400:00000200:0.0:1784948705.716712:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948705.716721:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948705.716727:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0339559 00000400:00000010:0.0:1784948705.716731:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f0339559. 00000100:00000001:0.0:1784948705.716738:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948705.716741:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948705.716746:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000095b72665 x1871651289646976 msgsize 224 00000100:00100000:0.0:1784948705.716753:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948705.716755:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948705.716778:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948705.716791:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:0.0:1784948705.716794:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000100:00000001:0.0:1784948705.716801:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948705.716810:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948705.716843:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948705.716849:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948705.716861:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948705.716866:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948705.716880:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948705.716892:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305645098 02000000:00000010:1.0:1784948705.716895:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007465fe15. 02000000:00000001:1.0:1784948705.716900:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948705.716903:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:0.0:1784948705.716914:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000010:1.0:1784948705.716946:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eadc14f. 00000400:00000010:1.0:1784948705.716964:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000100:00000001:0.0:1784948705.716972:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305645099 00000100:00000200:1.0:1784948705.716973:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289647104, portal 10 00000100:00000001:1.0:1784948705.716979:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000010:0.0:1784948705.716980:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:1.0:1784948705.716984:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451651456 : -115544257900160 : ffff96e9c18d5180) 00000100:00000001:0.0:1784948705.716991:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1784948705.716996:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948705.717002:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305645099 00000100:00000010:0.0:1784948705.717010:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:1.0:1784948705.717016:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948705.717017:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948705.717024:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305645099 00000100:00000040:1.0:1784948705.717025:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948705.717030:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000100:00000010:0.0:1784948705.717031:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:0.0:1784948705.717038:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000200:1.0:1784948705.717038:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289647104, offset 0 00000100:00000001:0.0:1784948705.717057:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948705.717062:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948705.717065:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948705.717070:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948705.717074:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948705.717079:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948705.717090:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948705.717102:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948705.717114:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948705.717121:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948705.717136:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3131:3131:240:4294967295] 192.168.202.137@tcp LPNI seq info [2038:2038:8:4294967295] 00000400:00000200:1.0:1784948705.717170:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948705.717185:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948705.717194:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000635133da. 00000800:00000200:1.0:1784948705.717208:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948705.717229:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948705.717240:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000635133da type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948705.717267:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948705.717279:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2021 usecs (3 jiffies) 00000100:00000001:1.0:1784948705.717288:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305645060 00000100:00000001:1.0:1784948705.717296:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305645098 00000100:00000001:1.0:1784948705.717307:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305645060 00000100:00000001:1.0:1784948705.717315:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305645099 00000100:00000001:1.0:1784948705.717322:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305645060 00000100:00000001:1.0:1784948705.717328:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305645099 00000100:00000001:1.0:1784948705.717360:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305645060 00000100:00000001:1.0:1784948705.717369:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305645099 00000100:00000001:1.0:1784948705.717381:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948705.717391:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.717452:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.717462:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948705.717604:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948705.717610:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000635133da. 00000400:00000200:3.0:1784948705.717617:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948705.717626:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948705.717635:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:3.0:1784948705.717638:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000100:00000001:3.0:1784948705.717645:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784948705.717652:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948705.717658:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948705.717670:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948705.717677:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948705.717681:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948705.717693:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948705.717701:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948705.717703:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948705.717706:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948705.717728:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948705.717753:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434800 00000400:00000200:0.0:1784948705.717769:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 149104 00000800:00000001:0.0:1784948705.717790:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948705.717802:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948705.717810:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948705.717822:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948705.717833:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948705.717837:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784948705.717842:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cdb12637. 00000100:00000040:0.0:1784948705.717847:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000cdb12637 x1871651289647104 msgsize 224 00000100:00100000:0.0:1784948705.717861:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948705.717884:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784948705.717901:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948705.717936:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948705.717941:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:0.0:1784948705.717949:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:1.0:1784948705.717951:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948705.717959:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948705.717963:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c7491352. 00000100:00000001:3.0:1784948705.717966:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948705.717973:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289647104 00000400:00000200:1.0:1784948705.717975:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 02000000:00000001:3.0:1784948705.717977:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000010:1.0:1784948705.717984:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:3.0:1784948705.717985:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948705.717989:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.717995:0:337121:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1784948705.717997:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948705.718004:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289647104 00000020:00000001:3.0:1784948705.718007:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:1.0:1784948705.718009:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784948705.718012:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784948705.718019:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948705.718025:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948705.718034:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784948705.718051:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784948705.718059:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948705.718059:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948705.718066:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000594c0c3d. 00000100:00000001:3.0:1784948705.718073:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948705.718081:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000594c0c3d. 00000020:00000010:3.0:1784948705.718111:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000100:00000001:1.0:1784948705.718115:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784948705.718119:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dc76901d. 00000100:00000001:1.0:1784948705.718121:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.718130:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:3.0:1784948705.718132:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000fc014090. 02000000:00000001:1.0:1784948705.718134:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948705.718140:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784948705.718147:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948705.718148:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948705.718156:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784948705.718157:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948705.718161:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00001000:1.0:1784948705.718167:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=27 v=5 (5 5 5 5) 00000100:00000001:3.0:1784948705.718168:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784948705.718182:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=27 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948705.718184:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948705.718191:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948705.718194:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948705.718195:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948705.718203:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948705.718205:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784948705.718210:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.718216:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948705.718222:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948705.718232:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948705.718236:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.718239:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948705.718248:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948705.718258:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948705.718260:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948705.718268:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948705.718284:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 661 00000100:00100000:1.0:1784948705.718289:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a8233e67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289646976:0@lo:400: 00000100:00000040:3.0:1784948705.718294:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948705.718306:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558291328 : -115544151260288 : ffff96e9c7e88380) 00000100:00000001:1.0:1784948705.718309:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.718318:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948705.718323:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000cdb12637 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:366/0 lens 224/0 e 0 to 0 dl 1784948716 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948705.718331:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a8233e67 x1871651289646976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948705.718381:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948705.718382:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948705.718430:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:3.0:1784948705.718432:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000594c0c3d. 02000000:00000010:1.0:1784948705.718435:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 00000100:00000001:3.0:1784948705.718440:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948705.718446:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000594c0c3d. 02000000:00000001:1.0:1784948705.718447:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948705.718452:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948705.718456:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00100000:3.0:1784948705.718457:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000cdb12637 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289647104:12345-192.168.202.137@tcp:400: 00000020:00000001:1.0:1784948705.718462:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948705.718469:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000008f5595bf. 00000100:00000200:3.0:1784948705.718470:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289647104 00000020:00000001:3.0:1784948705.718477:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:1.0:1784948705.718478:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1784948705.718481:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:1.0:1784948705.718482:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948705.718488:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a8233e67. 00000020:00000001:3.0:1784948705.718488:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.718498:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1784948705.718498:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948705.718503:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784948705.718504:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.718514:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948705.718514:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948705.718521:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.718524:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1784948705.718526:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948705.718529:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948705.718530:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948705.718538:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948705.718538:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.718544:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948705.718545:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948705.718552:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948705.718555:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948705.718559:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a4e83fe8. 02000000:00000001:3.0:1784948705.718562:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948705.718565:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948705.718567:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948705.718569:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289647104 00010000:00000001:3.0:1784948705.718572:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948705.718576:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000cdb12637 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:366/0 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948705.718584:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948705.718587:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948705.718589:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=22 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948705.718592:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948705.718594:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948705.718596:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784948705.718599:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948705.718601:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948705.718604:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948705.718606:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948705.718609:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000100:00000200:3.0:1784948705.718613:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289647104, offset 224 00000400:00000200:3.0:1784948705.718618:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948705.718626:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948705.718632:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3132:3132:240:4294967295] 192.168.202.137@tcp LPNI seq info [2039:2039:8:4294967295] 00000400:00000200:3.0:1784948705.718642:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948705.718647:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948705.718650:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000089a028d. 00000800:00000200:3.0:1784948705.718655:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948705.718661:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948705.718664:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000089a028d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948705.718680:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948705.718684:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948705.718686:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948705.718688:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948705.718690:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948705.718693:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cdb12637 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:366/0 lens 224/224 e 0 to 0 dl 1784948716 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948705.718703:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000cdb12637 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289647104:12345-192.168.202.137@tcp:400: Request processed in 323us (854us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948705.718712:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 661 00000100:00000040:3.0:1784948705.718715:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948705.718719:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948705.718721:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948705.718724:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dc76901d. 00000020:00000010:3.0:1784948705.718727:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000fc014090. 00000020:00000010:3.0:1784948705.718731:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000020:00000040:3.0:1784948705.718738:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948705.718741:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948705.718822:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948705.718828:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000089a028d. 00000400:00000200:0.0:1784948705.718833:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948705.718841:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948705.718843:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948705.718846:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000400:00000010:0.0:1784948705.718850:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000800:00000001:3.0:1784948705.718854:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948705.718855:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948705.718861:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948705.718862:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948705.718868:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948705.718876:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948705.718888:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434800 00000400:00000200:3.0:1784948705.718894:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8025 [1] + 224 00000800:00000001:3.0:1784948705.718900:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948705.718936:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948705.718939:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948705.718943:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948705.718947:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948705.718951:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948705.718961:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948705.718978:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948705.718984:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948705.718988:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.719024:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948705.719028:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948705.719032:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948705.719034:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948705.719036:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002eadc14f. 00000400:00000200:1.0:1784948705.719040:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:1.0:1784948705.719042:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000100:00000001:1.0:1784948705.719045:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948705.719050:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948705.719059:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948705.719067:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948705.719068:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.719070:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948705.719072:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948705.719074:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948705.719075:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948705.719077:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948705.719082:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=22 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948705.719084:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=22 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948705.719087:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948705.719088:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948705.719090:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948705.719094:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948705.719102:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.719104:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948705.719106:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948705.719108:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948705.719110:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948705.719113:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948705.719120:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948705.719122:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948705.719141:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948705.719144:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948705.719153:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289647104:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948705.719158:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.719160:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948705.719162:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b55e20f2 x1871651289647104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948705.719170:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948705.719171:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948705.719173:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007465fe15. 02000000:00000001:1.0:1784948705.719177:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948705.719178:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948705.719180:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948705.719182:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948705.719184:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 02000000:00000001:1.0:1784948705.719187:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948705.719188:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948705.719190:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b55e20f2. 00000100:00000001:1.0:1784948705.719193:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948705.719195:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.719197:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948705.719201:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948705.719203:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948705.719205:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948705.719207:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948705.759991:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305645099, at 4305645100 00000100:00000010:3.0:1784948705.760014:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000040:0.0:1784948705.760020:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305645098, at 4305645100 00000100:00000040:1.0:1784948705.760020:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305645099, at 4305645100 00000100:00000010:0.0:1784948705.760041:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:1.0:1784948705.760044:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000020:00000001:3.0:1784948706.548202:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948706.548210:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948706.548216:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948707.572087:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948707.572091:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948707.572094:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948708.596190:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948708.596196:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948708.596199:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948708.788095:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948708.788109:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948708.788111:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948708.788116:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948708.788119:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948708.788123:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948708.788125:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948709.620122:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948709.620127:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948709.620129:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948709.812004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948709.812017:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948709.812019:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948709.812024:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948709.812026:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948709.812030:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948709.812032:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948710.644186:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948710.644192:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948710.644198:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948710.836049:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1784948710.836054:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948710.836066:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948710.836067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948710.836073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784948710.836078:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b55e20f2. 00000100:00000001:0.0:1784948710.836080:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948710.836085:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1784948710.836088:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948710.836092:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948710.836098:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948710.836099:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948710.836106:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948710.836116:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948710.836127:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00000001:1.0:1784948710.836137:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948710.836151:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948710.836168:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948710.836172:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948710.836180:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b55e20f2] to pc [ptlrpcd_00_01+1] req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948710.836219:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948710.836223:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948710.836229:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948710.836234:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948710.836240:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948710.836247:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a8233e67. 00000020:00000040:1.0:1784948710.836252:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948710.836261:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948710.836267:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948710.836272:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948710.836273:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948710.836283:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948710.836286:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000008f5595bf. 00000100:00000001:1.0:1784948710.836299:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948710.836318:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948710.836339:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948710.836345:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948710.836387:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8233e67] to pc [ptlrpcd_00_02+2] req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948710.836431:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948710.836442:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10987) 00000100:00000001:3.0:1784948710.836471:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948710.836480:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948710.836486:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948710.836487:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948710.836496:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948710.836503:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948710.836547:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948710.836550:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948710.836556:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948710.836563:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948710.836574:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948710.836579:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a8233e67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289647360:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948710.836585:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948710.836590:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948710.836595:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948710.836603:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948710.836612:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005e186afc. 02000000:00000001:3.0:1784948710.836618:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948710.836633:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000400:00000010:3.0:1784948710.836662:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000100:00000200:3.0:1784948710.836672:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289647360, portal 10 00000100:00000001:3.0:1784948710.836688:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948710.836695:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451649664 : -115544257901952 : ffff96e9c18d4a80) 00000100:00000040:3.0:1784948710.836714:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948710.836742:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948710.836755:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948710.836760:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00000200:3.0:1784948710.836764:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289647360, offset 0 00000400:00000200:3.0:1784948710.836772:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948710.836788:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948710.836800:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948710.836808:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948710.836813:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948710.836818:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948710.836824:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3133:3133:240:4294967295] 192.168.202.137@tcp LPNI seq info [2040:2040:8:4294967295] 00000400:00000200:3.0:1784948710.836838:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948710.836844:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948710.836852:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000071cce8cf. 00000800:00000200:3.0:1784948710.836866:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948710.836892:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784948710.836948:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948710.836957:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289647232 02000000:00000001:0.0:1784948710.836965:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948710.836970:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948710.836977:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948710.836980:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000071cce8cf type 1, nob 320 niov 1 nkiov 1 02000000:00000001:0.0:1784948710.836988:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948710.837001:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289647232 00000020:00000001:0.0:1784948710.837012:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948710.837016:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948710.837020:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:3.0:1784948710.837028:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2023 usecs (3 jiffies) 00000020:00000001:0.0:1784948710.837030:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948710.837037:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948710.837041:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305650180 00000020:00000001:0.0:1784948710.837044:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948710.837054:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784948710.837057:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:0.0:1784948710.837062:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948710.837074:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000874e7bc2. 00000100:00000001:3.0:1784948710.837074:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948710.837082:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948710.837089:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000874e7bc2. 00000100:00000001:3.0:1784948710.837090:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305650180 00000100:00000010:3.0:1784948710.837102:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000010:0.0:1784948710.837103:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000100:00000001:3.0:1784948710.837115:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000010:0.0:1784948710.837121:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009bd302f2. 00000100:00000001:3.0:1784948710.837127:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305650180 00000020:00000010:0.0:1784948710.837134:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000010:3.0:1784948710.837138:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:3.0:1784948710.837150:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000040:0.0:1784948710.837160:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948710.837163:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305650180 00000100:00000001:0.0:1784948710.837170:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:3.0:1784948710.837173:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:0.0:1784948710.837175:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948710.837185:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948710.837189:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948710.837197:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948710.837203:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948710.837210:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948710.837218:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948710.837224:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948710.837235:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948710.837238:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948710.837247:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948710.837252:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948710.837259:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1784948710.837263:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 686 00000100:00000001:3.0:1784948710.837271:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948710.837273:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948710.837276:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948710.837283:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818136144 : -115538891415472 : ffff96eb016b6050) 00000100:00000001:3.0:1784948710.837284:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948710.837298:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948710.837301:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f911f57b x1871651289647232/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:371/0 lens 224/0 e 0 to 0 dl 1784948721 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948710.837327:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948710.837333:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000874e7bc2. 00000100:00000001:0.0:1784948710.837337:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948710.837341:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000874e7bc2. 00000100:00100000:0.0:1784948710.837348:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f911f57b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289647232:12345-0@lo:400: 00000100:00000200:0.0:1784948710.837379:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289647232 00000800:00000200:1.0:1784948710.837383:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1784948710.837384:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948710.837392:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000010:1.0:1784948710.837394:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000071cce8cf. 00000020:00000001:0.0:1784948710.837398:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948710.837404:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:1.0:1784948710.837406:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784948710.837409:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948710.837416:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:1.0:1784948710.837424:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784948710.837425:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948710.837435:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:1.0:1784948710.837436:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000020:00000001:0.0:1784948710.837443:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:1.0:1784948710.837446:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000020:00000001:0.0:1784948710.837451:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948710.837456:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948710.837459:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948710.837461:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000200:1.0:1784948710.837469:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948710.837471:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948710.837482:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000081e7de16. 02000000:00000001:0.0:1784948710.837494:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948710.837505:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948710.837517:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948710.837517:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000040:0.0:1784948710.837531:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289647232 00000100:00000040:1.0:1784948710.837532:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948710.837543:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948710.837558:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f911f57b x1871651289647232/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:371/0 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948710.837588:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948710.837600:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1784948710.837604:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948710.837610:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948710.837617:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=32 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948710.837624:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948710.837632:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948710.837638:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948710.837645:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948710.837650:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948710.837657:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948710.837668:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948710.837675:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00000200:0.0:1784948710.837681:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289647232, offset 224 00000400:00000200:0.0:1784948710.837694:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948710.837708:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948710.837725:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434880 00000400:00000200:0.0:1784948710.837735:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8039 [1] + 224 00000400:00000200:0.0:1784948710.837745:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948710.837754:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948710.837762:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948710.837776:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948710.837778:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948710.837786:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289647360 02000000:00000001:3.0:1784948710.837792:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948710.837795:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948710.837799:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948710.837807:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948710.837810:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948710.837813:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289647360 00000020:00000001:3.0:1784948710.837822:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948710.837828:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784948710.837834:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948710.837839:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948710.837843:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784948710.837847:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948710.837851:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948710.837852:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784948710.837858:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001f45750. 00000400:00000200:0.0:1784948710.837865:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000100:00000001:3.0:1784948710.837866:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948710.837873:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000020:00000010:3.0:1784948710.837877:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001f45750. 00000100:00000001:0.0:1784948710.837881:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948710.837888:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000010:3.0:1784948710.837895:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000859865aa. 00000100:00000001:1.0:1784948710.837896:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948710.837897:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948710.837935:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948710.837948:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:0.0:1784948710.837953:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:1.0:1784948710.837957:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:3.0:1784948710.837958:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dc76901d. 00010000:00000001:0.0:1784948710.837961:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:1.0:1784948710.837967:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000037df9850. 00000020:00000001:0.0:1784948710.837968:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948710.837973:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000fc014090. 00000020:00000001:0.0:1784948710.837975:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:1.0:1784948710.837983:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000100:00000040:0.0:1784948710.837988:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f911f57b x1871651289647232/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:371/0 lens 224/224 e 0 to 0 dl 1784948721 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948710.837994:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000100:00000040:3.0:1784948710.837994:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948710.838007:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948710.838011:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784948710.838014:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:0.0:1784948710.838021:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f911f57b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289647232:12345-0@lo:400: Request processed in 688us (1292us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948710.838023:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948710.838033:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948710.838038:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948710.838046:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00100000:0.0:1784948710.838047:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 686 00000100:00000001:3.0:1784948710.838051:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948710.838059:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948710.838068:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948710.838071:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948710.838075:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948710.838088:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009bd302f2. 00000100:00000200:1.0:1784948710.838089:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948710.838098:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948710.838108:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000100:00000001:3.0:1784948710.838110:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000040:0.0:1784948710.838122:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948710.838122:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948710.838126:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1784948710.838136:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784948710.838137:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784948710.838144:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000081e7de16. 00000100:00000001:3.0:1784948710.838148:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948710.838149:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948710.838157:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:1.0:1784948710.838158:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948710.838161:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948710.838165:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948710.838166:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948710.838175:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948710.838179:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:1.0:1784948710.838186:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948710.838191:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f301fed7. 00000100:00001000:1.0:1784948710.838200:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=32 v=5 (5 5 5 5) 00000400:00000010:0.0:1784948710.838205:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000be708ecf. 00000100:00001000:1.0:1784948710.838224:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=32 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948710.838232:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948710.838235:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948710.838247:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784948710.838253:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948710.838258:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948710.838269:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948710.838306:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948710.838316:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948710.838323:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948710.838336:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948710.838394:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289647232:0@lo:400: 00000100:00000001:1.0:1784948710.838408:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948710.838417:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948710.838425:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b55e20f2 x1871651289647232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948710.838457:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948710.838466:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948710.838473:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:1.0:1784948710.838485:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948710.838492:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948710.838498:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784948710.838508:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948710.838514:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 02000000:00000001:1.0:1784948710.838527:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948710.838533:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948710.838538:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b55e20f2. 00000100:00000001:1.0:1784948710.838546:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948710.838552:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948710.838557:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948710.838571:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948710.838578:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948710.838584:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948710.838592:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948710.838864:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948710.838872:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007b2ad4bf. 00000400:00000200:3.0:1784948710.838881:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948710.838893:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948710.838895:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948710.838901:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000800:00000001:1.0:1784948710.838940:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948710.838947:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948710.838953:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948710.838965:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000400:00000200:1.0:1784948710.838976:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948710.838980:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948710.838985:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948710.839000:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434900 00000400:00000200:1.0:1784948710.839014:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8041 [1] + 224 00000800:00000001:1.0:1784948710.839025:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948710.839042:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948710.839045:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948710.839050:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948710.839056:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948710.839060:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948710.839071:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948710.839094:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948710.839102:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948710.839105:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948710.839143:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948710.839148:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948710.839153:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948710.839155:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948710.839157:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000400:00000200:3.0:1784948710.839164:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:3.0:1784948710.839166:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:3.0:1784948710.839169:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948710.839175:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948710.839187:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948710.839196:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948710.839198:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948710.839200:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948710.839203:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948710.839204:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948710.839206:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948710.839209:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948710.839214:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=27 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948710.839218:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=27 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948710.839221:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948710.839223:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948710.839226:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948710.839230:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948710.839238:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948710.839242:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948710.839243:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948710.839247:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948710.839248:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948710.839252:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948710.839260:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948710.839263:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948710.839264:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948710.839267:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948710.839276:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a8233e67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289647360:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948710.839281:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948710.839283:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948710.839285:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a8233e67 x1871651289647360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948710.839293:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948710.839295:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948710.839296:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000005e186afc. 02000000:00000001:3.0:1784948710.839302:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948710.839304:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948710.839305:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948710.839307:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948710.839309:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000008f5595bf. 02000000:00000001:3.0:1784948710.839312:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948710.839314:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948710.839315:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a8233e67. 00000100:00000001:3.0:1784948710.839319:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948710.839320:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948710.839322:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948710.839327:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948710.839329:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948710.839331:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948710.839333:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948710.880027:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305650219, at 4305650220 00000100:00000040:1.0:1784948710.880034:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305650219, at 4305650220 00000100:00000040:3.0:1784948710.880035:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305650219, at 4305650220 00000100:00000010:0.0:1784948710.880045:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:3.0:1784948710.880051:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:1.0:1784948710.880062:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000020:00000001:1.0:1784948711.668123:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948711.668130:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948711.668134:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948712.692163:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948712.692169:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948712.692173:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948713.716146:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948713.716153:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948713.716157:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948714.740212:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948714.740219:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948714.740223:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948715.764191:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948715.764196:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948715.764200:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948715.956033:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948715.956048:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948715.956061:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b55e20f2. 00000100:00000001:0.0:1784948715.956064:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948715.956070:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000040:1.0:1784948715.956070:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948715.956077:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948715.956082:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1784948715.956086:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948690/real 1784948690] req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948715 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948715.956089:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948715.956099:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000400:00000001:0.0:1784948715.956107:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784948715.956107:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784948715.956110:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948715.956114:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dbb46726. 00000100:00000040:1.0:1784948715.956116:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784948715.956122:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:0.0:1784948715.956127:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784948715.956130:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948715.956134:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956135:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948715.956140:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948715 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948715.956141:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b55e20f2] to pc [ptlrpcd_00_03+3] req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948715.956155:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948715 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948715.956166:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948715.956169:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.956171:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956172:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1784948715.956174:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:0.0:1784948715.956176:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1784948715.956180:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1784948715.956183:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948715 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948715.956185:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948715.956191:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948715.956194:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784948715.956197:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007fef3945. 00000020:00000040:1.0:1784948715.956201:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948715.956203:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948715.956204:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948715.956204:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948715.956208:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948715.956208:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948715.956212:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956214:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784948715.956214:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:0.0:1784948715.956217:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1784948715.956219:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d104522b. 00000100:00000001:3.0:1784948715.956221:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948715.956224:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.956224:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948715.956232:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784948715.956233:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784948715.956234:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948715 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948715.956239:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948715.956250:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948715.956257:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948715.956258:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784948715.956258:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948715.956263:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948715.956264:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948715.956265:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948715.956270:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007fef3945] to pc [ptlrpcd_00_00+0] req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948715.956271:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948715.956273:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948715.956277:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:3.0:1784948715.956279:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948715.956281:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948715.956288:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1784948715.956299:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948715.956343:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10992) 00000100:00000001:3.0:1784948715.956346:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1784948715.956349:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:3.0:1784948715.956351:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956355:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1784948715.956359:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1784948715.956360:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000040:0.0:1784948715.956364:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948715 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948715.956366:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1784948715.956368:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948715.956372:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948715.956377:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1784948715.956377:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.956386:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1784948715.956389:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289647488:0@lo:400: 00000100:00080000:1.0:1784948715.956393:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10962 00000100:00100000:0.0:1784948715.956395:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000adacbaf0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289646208:192.168.202.137@tcp:8: 00000100:00000001:3.0:1784948715.956403:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:1.0:1784948715.956405:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 02000000:00000001:3.0:1784948715.956412:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1784948715.956413:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 10961 02000000:00000001:3.0:1784948715.956418:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948715.956423:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 02000000:00000001:3.0:1784948715.956424:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784948715.956431:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000010:3.0:1784948715.956432:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000859865aa. 00000100:00000001:1.0:1784948715.956436:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784948715.956437:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948715.956442:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000400:00000010:3.0:1784948715.956446:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000100:00000001:1.0:1784948715.956447:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000001:1.0:1784948715.956454:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948715.956456:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000010:3.0:1784948715.956457:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000020:00000001:1.0:1784948715.956459:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948715.956465:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000200:3.0:1784948715.956465:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289647488, portal 25 00000020:00000001:1.0:1784948715.956470:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:3.0:1784948715.956471:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948715.956474:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451651456 : -115544257900160 : ffff96e9c18d5180) 00000020:00000001:1.0:1784948715.956477:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:3.0:1784948715.956483:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948715.956484:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948715.956490:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948715.956495:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948715.956502:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:3.0:1784948715.956504:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1784948715.956508:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000040:3.0:1784948715.956512:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00080000:1.0:1784948715.956516:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000010:3.0:1784948715.956517:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00000200:3.0:1784948715.956524:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289647488, offset 0 00000100:00080000:1.0:1784948715.956525:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:1.0:1784948715.956531:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956534:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948715.956535:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784948715.956536:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1784948715.956539:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784948715.956543:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948715.956545:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000adacbaf0 x1871651289646208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948715 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948715.956547:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.956550:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1784948715.956573:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948715.956575:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000010:1.0:1784948715.956582:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e8f14423. 00000100:00000001:0.0:1784948715.956586:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000040:1.0:1784948715.956586:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=4 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784948715.956590:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1784948715.956591:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000010:0.0:1784948715.956593:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000371d55ed. 02000000:00000001:1.0:1784948715.956595:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948715.956596:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434980 00000100:00000001:1.0:1784948715.956599:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948715.956601:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948715.956605:0:337118:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:1.0:1784948715.956605:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1784948715.956608:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7649 [2] + 0 00000020:00000040:0.0:1784948715.956610:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:1.0:1784948715.956610:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948715.956617:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000010:3.0:1784948715.956619:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000055895529. 02000000:00000010:1.0:1784948715.956621:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000e6df9f3d. 02000000:00000010:0.0:1784948715.956622:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000fd4fe1c8. 00000400:00000200:3.0:1784948715.956628:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000008f3d6065 02000000:00000001:0.0:1784948715.956630:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1784948715.956631:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948715.956636:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948715.956638:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000adacbaf0. 00000100:00100000:1.0:1784948715.956639:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1784948715.956641:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948715.956643:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948715.956650:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948715.956655:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948715.956657:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.956657:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948715.956662:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956665:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1784948715.956665:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008f3d6065 00000100:00000040:1.0:1784948715.956670:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e8f14423] to pc [ptlrpcd_rcv+-1] req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948715.956671:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000008f3d6065. 00000100:00000001:0.0:1784948715.956672:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956680:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948715.956683:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784948715.956684:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948715.956688:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784948715.956689:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956690:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948715.956694:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948715.956697:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948715.956699:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000046e93177 x1871651289647488 msgsize 224 00000100:00000040:1.0:1784948715.956701:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10992) 00000100:00000040:0.0:1784948715.956705:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948715.956730:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784948715.956737:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948715.956743:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948715.956746:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948715.956751:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948715.956751:0:337121:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:0.0:1784948715.956756:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948715.956760:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948715.956764:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000100:00100000:0.0:1784948715.956767:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289647744:1.2.3.4@tcp:8: 00000400:00000010:3.0:1784948715.956769:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00000001:0.0:1784948715.956777:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948715.956777:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948715.956784:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.956785:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1784948715.956786:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948715.956790:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.956790:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956798:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948715.956802:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948715.956803:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948715.956809:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948715.956811:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948715.956814:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.956821:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948715.956821:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948715.956827:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1784948715.956829:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948715.956830:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948715.956832:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948715.956832:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948715.956837:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.956839:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948715.956840:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.956842:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.956843:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.956849:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948715.956850:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305655338 00000100:00000001:1.0:1784948715.956851:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948715.956857:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1784948715.956860:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000040:1.0:1784948715.956867:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948715.956869:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948715.956878:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305655338 00000100:00000010:3.0:1784948715.956887:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000afdabce. 00000100:00000001:3.0:1784948715.956894:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948715.956900:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948715.956901:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305655338 00000100:00000010:3.0:1784948715.956907:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:3.0:1784948715.956915:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948715.956924:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305655339 00000100:00000001:1.0:1784948715.956928:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948715.956929:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 02000000:00000001:1.0:1784948715.956937:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948715.956938:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:1.0:1784948715.956943:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948715.956947:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948715.956948:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948715.956957:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007fef3945 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289647616:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948715.956965:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948715.956970:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948715.956972:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948715.956978:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948715.956978:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948715.956986:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000003b287909. 00000100:00000001:3.0:1784948715.956986:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948715.957010:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948715.957011:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948715.957021:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eadc14f. 00000100:00000001:3.0:1784948715.957028:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784948715.957031:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000100:00000001:3.0:1784948715.957034:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948715.957039:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289647616, portal 10 00000100:00000001:1.0:1784948715.957046:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948715.957049:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451660416 : -115544257891200 : ffff96e9c18d7480) 00000100:00000040:1.0:1784948715.957060:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948715.957086:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948715.957099:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948715.957107:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000100:00000200:1.0:1784948715.957222:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289647616, offset 0 00000400:00000200:1.0:1784948715.957243:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948715.957258:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948715.957270:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948715.957277:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948715.957284:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948715.957289:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948715.957295:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3135:3135:240:4294967295] 192.168.202.137@tcp LPNI seq info [2042:2042:8:4294967295] 00000400:00000200:1.0:1784948715.957324:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948715.957334:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948715.957368:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003a8a98bc. 00000800:00000200:1.0:1784948715.957379:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948715.957394:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948715.957402:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003a8a98bc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948715.957421:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948715.957430:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2025 usecs (3 jiffies) 00000800:00000200:0.0:1784948715.957543:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948715.957550:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000400:00000200:0.0:1784948715.957575:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948715.957589:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305655300 00000400:00000200:0.0:1784948715.957590:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948715.957599:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305655338 00000100:00000001:3.0:1784948715.957600:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784948715.957602:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000100:00000001:3.0:1784948715.957606:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948715.957607:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305655300 00000400:00000010:0.0:1784948715.957609:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000100:00000001:1.0:1784948715.957614:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305655338 00000100:00000001:0.0:1784948715.957616:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948715.957620:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305655300 00000400:00000001:3.0:1784948715.957622:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784948715.957623:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305655338 00000100:00000200:0.0:1784948715.957624:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784948715.957627:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784948715.957630:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305655300 00000400:00000010:3.0:1784948715.957632:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000100:00000001:1.0:1784948715.957635:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305655339 00000100:00000001:1.0:1784948715.957641:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948715.957643:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000100:00000001:0.0:1784948715.957645:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948715.957645:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948715.957649:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:1.0:1784948715.957651:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948715.957653:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948715.957665:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948715.957671:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948715.957677:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784948715.957689:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948715.957703:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948715.957707:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784948715.957709:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948715.957716:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948715.957727:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948715.957752:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948715.957758:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948715.957762:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948715.957762:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948715.957770:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434a00 00000100:00000001:3.0:1784948715.957771:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948715.957775:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948715.957778:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948715.957780:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948715.957786:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 149552 02000000:00000001:3.0:1784948715.957786:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948715.957797:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:3.0:1784948715.957799:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=37 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948715.957804:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=37 v=5 (1 1 1 1) 00000800:00000001:0.0:1784948715.957809:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948715.957809:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948715.957813:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948715.957816:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:3.0:1784948715.957818:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784948715.957822:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948715.957826:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948715.957829:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:3.0:1784948715.957850:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948715.957854:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948715.957859:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784948715.957859:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948715.957866:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:0.0:1784948715.957869:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000efa2f6d. 00000100:00000001:3.0:1784948715.957869:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948715.957875:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948715.957877:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000000efa2f6d x1871651289647616 msgsize 224 00000100:00100000:3.0:1784948715.957890:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b55e20f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289647488:0@lo:400: 00000100:00100000:0.0:1784948715.957894:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784948715.957901:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948715.957905:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948715.957909:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:3.0:1784948715.957909:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b55e20f2 x1871651289647488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948715.957919:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948715.957924:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948715.957935:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948715.957939:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948715.957942:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000859865aa. 02000000:00000001:3.0:1784948715.957947:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948715.957948:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948715.957950:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784948715.957952:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948715.957954:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 02000000:00000001:3.0:1784948715.957957:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948715.957959:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948715.957960:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b55e20f2. 00000100:00000001:3.0:1784948715.957964:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948715.957965:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948715.957967:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948715.957973:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948715.957974:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948715.957977:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948715.957979:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.957985:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948715.957989:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289647616 02000000:00000001:1.0:1784948715.957992:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948715.957994:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948715.957997:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948715.958000:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948715.958003:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289647616 00000020:00000001:1.0:1784948715.958005:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948715.958006:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948715.958008:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948715.958009:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948715.958012:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948715.958013:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948715.958016:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948715.958019:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948715.958021:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948715.958023:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000010:1.0:1784948715.958027:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007465fe15. 00000020:00000010:1.0:1784948715.958030:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cf558f26. 00000020:00000010:1.0:1784948715.958033:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:1.0:1784948715.958038:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948715.958040:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948715.958041:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948715.958043:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.958046:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948715.958048:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948715.958049:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.958059:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.958066:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948715.958068:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948715.958071:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 663 00000100:00000040:1.0:1784948715.958074:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948715.958076:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558300288 : -115544151251328 : ffff96e9c7e8a680) 00000100:00000040:1.0:1784948715.958080:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000efa2f6d x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:376/0 lens 224/0 e 0 to 0 dl 1784948726 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948715.958109:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948715.958111:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948715.958113:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948715.958114:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00100000:1.0:1784948715.958118:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000efa2f6d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289647616:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948715.958121:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289647616 00000020:00000001:1.0:1784948715.958122:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948715.958125:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948715.958126:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948715.958127:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948715.958129:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948715.958131:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948715.958133:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948715.958135:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948715.958136:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948715.958137:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948715.958139:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948715.958142:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948715.958143:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948715.958145:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000371d55ed. 02000000:00000001:1.0:1784948715.958147:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.958149:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948715.958150:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948715.958151:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289647616 00010000:00000001:1.0:1784948715.958154:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948715.958156:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000efa2f6d x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:376/0 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948715.958162:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948715.958163:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948715.958165:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=32 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948715.958168:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948715.958170:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948715.958172:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948715.958174:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948715.958175:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.958177:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948715.958179:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948715.958181:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000100:00000200:1.0:1784948715.958183:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289647616, offset 224 00000400:00000200:1.0:1784948715.958187:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948715.958192:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948715.958196:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3136:3136:240:4294967295] 192.168.202.137@tcp LPNI seq info [2043:2043:8:4294967295] 00000400:00000200:1.0:1784948715.958203:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948715.958207:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948715.958210:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000062b4a223. 00000800:00000200:1.0:1784948715.958213:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948715.958217:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948715.958219:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000062b4a223 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948715.958231:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948715.958235:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948715.958238:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948715.958239:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948715.958242:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948715.958246:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000efa2f6d x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:376/0 lens 224/224 e 0 to 0 dl 1784948726 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948715.958257:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000efa2f6d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289647616:12345-192.168.202.137@tcp:400: Request processed in 163us (379us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948715.958266:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 663 00000100:00000040:1.0:1784948715.958269:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948715.958271:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948715.958273:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948715.958277:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cf558f26. 00000020:00000010:1.0:1784948715.958280:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948715.958284:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007465fe15. 00000020:00000040:1.0:1784948715.958288:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948715.958291:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948715.958364:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948715.958367:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000062b4a223. 00000400:00000200:0.0:1784948715.958369:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948715.958373:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948715.958376:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:0.0:1784948715.958378:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000100:00000001:0.0:1784948715.958381:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948715.958383:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948715.958486:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948715.958489:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948715.958492:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948715.958494:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948715.958495:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002eadc14f. 00000400:00000200:1.0:1784948715.958498:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:1.0:1784948715.958499:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000100:00000001:1.0:1784948715.958501:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948715.958505:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948715.958511:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948715.958517:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948715.958518:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.958519:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948715.958521:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948715.958522:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948715.958523:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948715.958524:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948715.958527:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=32 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948715.958530:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=32 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948715.958532:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948715.958533:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948715.958534:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948715.958536:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948715.958541:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.958544:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948715.958545:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948715.958547:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948715.958550:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948715.958553:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948715.958562:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948715.958565:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948715.958566:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948715.958569:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948715.958598:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007fef3945 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289647616:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948715.958602:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.958605:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948715.958608:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007fef3945 x1871651289647616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948731 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948715.958615:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948715.958617:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948715.958619:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000003b287909. 02000000:00000001:1.0:1784948715.958623:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948715.958624:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948715.958626:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948715.958629:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948715.958631:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d104522b. 02000000:00000001:1.0:1784948715.958634:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948715.958636:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948715.958637:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007fef3945. 00000100:00000001:1.0:1784948715.958640:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948715.958641:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.958643:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948715.958647:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948715.958649:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948715.958651:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948715.958653:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948715.998981:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305655338, at 4305655339 00000100:00000040:1.0:1784948715.998981:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305655338, at 4305655339 00000100:00000010:0.0:1784948715.998999:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:1.0:1784948715.999000:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000afdabce. 00000100:00000040:3.0:1784948716.000044:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305655339, at 4305655340 00000100:00000010:3.0:1784948716.000056:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000020:00000001:3.0:1784948716.788068:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948716.788073:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948716.788076:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948716.980021:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948716.980029:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948716.980045:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948715/real 0] req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948716 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.980062:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.980066:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948716.980068:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948716.980073:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948716 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.980081:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.980091:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948716.980093:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.980097:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948716.980100:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948716.980103:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948716.980107:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948716 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.980115:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784948716.980120:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.980122:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948716.980124:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.980126:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784948716.980129:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948716.980133:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784948716.980164:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784948716.980168:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784948716.980173:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948716 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948716.980182:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289647744:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784948716.980191:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.980194:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948716.980199:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e8f14423 x1871651289647744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948716 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.980209:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784948716.980213:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948716.980216:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:3.0:1784948716.980217:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1784948716.980220:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948716.980227:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000e6df9f3d. 00000100:00000001:3.0:1784948716.980228:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784948716.980238:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cca13cd1. 02000000:00000001:0.0:1784948716.980239:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948716.980244:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:3.0:1784948716.980247:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000010:0.0:1784948716.980248:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e8f14423. 00000100:00000001:3.0:1784948716.980257:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948716.980259:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1784948716.980262:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784948716.980264:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.980270:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948716.980270:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.980277:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948716.980280:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948716.980282:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e3ed36ac. 00000100:00000001:0.0:1784948716.980284:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948716.980289:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948716.980295:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948716.980339:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948716.980352:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948716.980354:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948716.980360:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cca13cd1] to pc [ptlrpcd_00_01+1] req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948716.980392:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948716.980395:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948716.980400:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948716.980403:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784948716.980406:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784948716.980411:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784948716.980418:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10962 00000100:00000001:0.0:1784948716.980420:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948716.980425:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948716.980429:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1784948716.980430:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000040:0.0:1784948716.980440:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1784948716.980440:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 10987 00000400:00000200:3.0:1784948716.980455:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:3.0:1784948716.980465:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1784948716.980467:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948716.980472:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948716.980473:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:0.0:1784948716.980478:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948716.980479:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784948716.980484:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948716.980488:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:0.0:1784948716.980491:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948716.980494:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:0.0:1784948716.980499:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000cca13cd1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289647872:0@lo:400: 00000020:00000001:3.0:1784948716.980503:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948716.980506:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1784948716.980507:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:3.0:1784948716.980513:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:0.0:1784948716.980515:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1784948716.980517:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:0.0:1784948716.980521:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948716.980522:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:0.0:1784948716.980526:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1784948716.980528:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:3.0:1784948716.980533:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000010:0.0:1784948716.980534:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eff166f6. 00000100:00000001:3.0:1784948716.980538:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784948716.980540:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948716.980545:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948716.980549:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000010:0.0:1784948716.980550:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000005ecef72. 00000020:00000040:3.0:1784948716.980552:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:3.0:1784948716.980562:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000400:00000010:0.0:1784948716.980564:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00080000:3.0:1784948716.980566:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:3.0:1784948716.980572:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948716.980576:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1784948716.980577:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289647872, portal 25 00000100:00000001:0.0:1784948716.980582:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948716.980585:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482518016 : -115544227033600 : ffff96e9c3644e00) 02000000:00000001:3.0:1784948716.980585:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948716.980594:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948716.980598:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948716.980598:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1784948716.980603:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948716.980615:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005dccb977. 00000100:00000001:0.0:1784948716.980616:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1784948716.980622:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:0.0:1784948716.980629:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784948716.980633:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1784948716.980635:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 02000000:00000001:3.0:1784948716.980637:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948716.980643:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289647872, offset 0 00000100:00000001:3.0:1784948716.980645:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948716.980651:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948716.980655:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948716.980657:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:3.0:1784948716.980666:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000900bf136. 00000100:00000001:3.0:1784948716.980672:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948716.980683:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:3.0:1784948716.980683:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948716.980698:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948716.980700:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948716.980706:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434b00 00000100:00000040:3.0:1784948716.980712:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005dccb977] to pc [ptlrpcd_rcv+-1] req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948716.980722:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x76e9 [2] + 0 00000400:00000010:0.0:1784948716.980733:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000ef44eaa. 00000400:00000200:0.0:1784948716.980740:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000008f8ece10 00000100:00000001:3.0:1784948716.980745:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948716.980749:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1784948716.980752:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948716.980753:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784948716.980760:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000095d61ef9. 00000400:00000200:0.0:1784948716.980768:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000040:3.0:1784948716.980768:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948716.980776:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948716.980781:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948716.980783:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948716.980786:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008f8ece10 02000000:00000010:3.0:1784948716.980790:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a219b995. 00000400:00000010:0.0:1784948716.980794:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000008f8ece10. 00000100:00000001:3.0:1784948716.980796:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.980809:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784948716.980813:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000095d61ef9 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1784948716.980815:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948716.980827:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000470139eb x1871651289647872 msgsize 224 00000100:00000001:3.0:1784948716.980829:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948716.980834:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948716.980844:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095d61ef9] to pc [ptlrpcd_00_02+2] req@0000000095d61ef9 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948716.980847:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948716.980855:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948716.980873:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948716.980881:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784948716.980886:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000100:00000001:3.0:1784948716.980890:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948716.980891:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000100:00000001:1.0:1784948716.980891:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948716.980900:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948716.980901:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1784948716.980938:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10994) 00000100:00000200:0.0:1784948716.980945:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948716.980951:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.980987:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948716.980995:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784948716.980996:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948716.981001:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948716.981012:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784948716.981015:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948716.981017:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948716.981023:0:337120:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1784948716.981025:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948716.981033:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289648000:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784948716.981034:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948716.981045:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948716.981047:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305656363 02000000:00000001:1.0:1784948716.981051:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948716.981057:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948716.981058:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 02000000:00000001:1.0:1784948716.981066:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948716.981074:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000010:1.0:1784948716.981075:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000e6df9f3d. 02000000:00000001:1.0:1784948716.981085:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.981088:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305656363 00000400:00000010:1.0:1784948716.981098:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eadc14f. 00000100:00000010:0.0:1784948716.981099:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:0.0:1784948716.981110:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000400:00000010:1.0:1784948716.981113:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000100:00000001:0.0:1784948716.981117:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305656363 00000100:00000001:3.0:1784948716.981122:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784948716.981124:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289648000, portal 4 00000100:00000010:0.0:1784948716.981128:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:3.0:1784948716.981131:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948716.981134:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948716.981141:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784948716.981141:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482519808 : -115544227031808 : ffff96e9c3645500) 00000100:00000001:0.0:1784948716.981153:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305656363 00000100:00000010:0.0:1784948716.981159:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000040:1.0:1784948716.981165:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.981186:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948716.981200:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948716.981209:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.981218:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948716.981220:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948716.981225:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948716.981236:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948716.981240:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948716.981245:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000010:1.0:1784948716.981247:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000100:00000001:0.0:1784948716.981250:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784948716.981254:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289648000, offset 0 00000100:00000001:0.0:1784948716.981255:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948716.981261:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.981270:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948716.981273:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948716.981293:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784948716.981298:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948716.981325:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289647872 00000400:00000200:1.0:1784948716.981328:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784948716.981331:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948716.981336:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784948716.981339:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784948716.981340:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948716.981351:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784948716.981352:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948716.981362:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289647872 00000400:00000200:1.0:1784948716.981365:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:0.0:1784948716.981369:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948716.981374:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:1.0:1784948716.981375:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3137:3137:240:4294967295] 192.168.202.137@tcp LPNI seq info [2044:2044:8:4294967295] 00000020:00000001:0.0:1784948716.981380:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948716.981385:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784948716.981388:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948716.981394:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784948716.981402:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948716.981403:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948716.981411:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076985c84. 00000800:00000200:1.0:1784948716.981417:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948716.981420:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948716.981427:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076985c84. 00000800:00000010:1.0:1784948716.981429:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d104522b. 00000020:00000010:0.0:1784948716.981438:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000069d2b91d. 00000800:00000200:1.0:1784948716.981445:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000010:0.0:1784948716.981450:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e5e96922. 00000020:00000010:0.0:1784948716.981458:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000800:00000200:1.0:1784948716.981464:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:0.0:1784948716.981473:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000200:1.0:1784948716.981475:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d104522b type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1784948716.981488:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948716.981493:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948716.981498:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948716.981508:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.981510:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948716.981523:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:0.0:1784948716.981533:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948716.981534:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305656363 00000100:00000001:1.0:1784948716.981546:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305656363 00000100:00000001:1.0:1784948716.981553:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305656363 00000100:00000001:0.0:1784948716.981554:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948716.981557:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305656363 00000100:00000001:0.0:1784948716.981560:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948716.981567:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305656363 00000100:00100000:0.0:1784948716.981577:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 688 00000100:00000001:1.0:1784948716.981578:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305656363 00000100:00000040:0.0:1784948716.981586:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948716.981587:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305656363 00000100:00000001:0.0:1784948716.981596:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818134096 : -115538891417520 : ffff96eb016b5850) 00000100:00000001:1.0:1784948716.981596:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305656363 00000100:00000001:1.0:1784948716.981606:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948716.981611:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000470139eb x1871651289647872/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:377/0 lens 224/0 e 0 to 0 dl 1784948727 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948716.981613:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948716.981623:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948716.981630:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.981637:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948716.981641:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784948716.981642:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076985c84. 00000100:00000001:1.0:1784948716.981647:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1784948716.981651:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948716.981654:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784948716.981658:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076985c84. 00000100:00000001:1.0:1784948716.981660:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948716.981667:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948716.981675:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000470139eb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289647872:12345-0@lo:400: 00000100:00000001:1.0:1784948716.981677:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948716.981689:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289647872 00000800:00000200:3.0:1784948716.981691:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1784948716.981695:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000010:3.0:1784948716.981700:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d104522b. 00000020:00000001:0.0:1784948716.981703:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948716.981709:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948716.981711:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784948716.981715:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948716.981720:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:3.0:1784948716.981725:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784948716.981728:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:3.0:1784948716.981736:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000020:00000001:0.0:1784948716.981737:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948716.981744:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000020:00000001:0.0:1784948716.981747:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784948716.981752:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784948716.981754:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948716.981761:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948716.981762:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948716.981773:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948716.981782:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948716.981789:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784948716.981793:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:3.0:1784948716.981797:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948741 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948716.981804:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000081e7de16. 00000100:00000001:3.0:1784948716.981808:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948716.981815:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948716.981816:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948716.981827:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948716.981829:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784948716.981831:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948716.981838:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289647872 00010000:00000001:0.0:1784948716.981843:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:3.0:1784948716.981844:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:0.0:1784948716.981855:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000470139eb x1871651289647872/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:377/0 lens 224/224 e 0 to 0 dl 1784948727 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948716.981855:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:0.0:1784948716.981872:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948716.981874:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948716.981878:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=38 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948716.981883:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948716.981885:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948716.981888:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948716.981891:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948716.981894:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.981898:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948716.981902:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948716.981956:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 00000100:00000200:0.0:1784948716.981961:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289647872, offset 224 00000400:00000200:0.0:1784948716.981968:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948716.981980:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948716.981989:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434b00 00000400:00000200:0.0:1784948716.981997:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8071 [1] + 224 00000800:00000001:3.0:1784948716.981999:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948716.982005:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948716.982018:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948716.982019:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948716.982032:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948716.982042:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948716.982043:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6434b80 00000400:00000100:3.0:1784948716.982060:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289648000 offset 0 length 520: 4 00000100:00000040:0.0:1784948716.982064:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948716.982068:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000f4826b64 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:3.0:1784948716.982078:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:3.0:1784948716.982097:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948716.982098:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948716.982111:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000400:00000010:0.0:1784948716.982114:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000100:00000001:0.0:1784948716.982120:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948716.982123:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948716.982129:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948716.982133:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1784948716.982134:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784948716.982144:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784948716.982145:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000800:00000010:1.0:1784948716.982145:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e6a3ade8. 00000100:00000001:3.0:1784948716.982149:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784948716.982152:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948716.982161:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:1.0:1784948716.982161:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000001:3.0:1784948716.982162:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948716.982191:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1784948716.982195:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000470139eb x1871651289647872/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:377/0 lens 224/224 e 0 to 0 dl 1784948727 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948716.982198:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000005ecef72. 00000400:00000200:1.0:1784948716.982204:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948716.982208:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:3.0:1784948716.982215:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000100:00000001:3.0:1784948716.982223:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1784948716.982225:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000100:00000200:3.0:1784948716.982234:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948716.982235:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000100:00100000:0.0:1784948716.982242:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000470139eb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289647872:12345-0@lo:400: Request processed in 598us (1408us total) trans 0 rc 0/0 00000100:00000001:1.0:1784948716.982250:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784948716.982263:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000095d61ef9 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948716.982272:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 688 00000100:00000040:0.0:1784948716.982286:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:3.0:1784948716.982291:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.982295:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948716.982321:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948716.982324:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:1.0:1784948716.982332:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000095d61ef9 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948732 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948716.982333:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e5e96922. 00000020:00000010:0.0:1784948716.982345:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784948716.982347:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948716.982353:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948716.982356:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948716.982357:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000069d2b91d. 00000100:00000001:3.0:1784948716.982358:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948716.982364:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948716.982365:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948716.982369:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:0.0:1784948716.982371:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948716.982375:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948716.982380:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948716.982382:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784948716.982389:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000081e7de16. 00000100:00001000:3.0:1784948716.982391:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=38 v=5 (5 5 5 5) 02000000:00000001:0.0:1784948716.982399:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:3.0:1784948716.982402:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=38 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948716.982404:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948716.982413:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000200:0.0:1784948716.982418:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784948716.982419:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948716.982428:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:0.0:1784948716.982429:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000b3b2e3c. 00010000:00000001:3.0:1784948716.982433:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948716.982441:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948716.982442:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a21d030b. 00000100:00000040:3.0:1784948716.982452:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948716.982473:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948716.982477:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948716.982480:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948716.982485:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948716.982498:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000cca13cd1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289647872:0@lo:400: 00000100:00000001:3.0:1784948716.982506:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948716.982513:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948716.982519:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000cca13cd1 x1871651289647872/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.982522:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948716.982531:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289648128 00000100:00000001:3.0:1784948716.982535:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948716.982536:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948716.982542:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:3.0:1784948716.982543:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1784948716.982546:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948716.982547:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eff166f6. 02000000:00000001:0.0:1784948716.982553:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948716.982556:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:0.0:1784948716.982560:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289648128 00000020:00000001:3.0:1784948716.982561:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948716.982566:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948716.982567:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948716.982573:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784948716.982577:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:0.0:1784948716.982578:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:3.0:1784948716.982585:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e3ed36ac. 00000020:00000040:0.0:1784948716.982587:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:3.0:1784948716.982593:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1784948716.982597:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:3.0:1784948716.982598:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1784948716.982603:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000010:3.0:1784948716.982604:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000cca13cd1. 00000100:00000001:0.0:1784948716.982612:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948716.982614:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948716.982617:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948716.982619:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076985c84. 00000100:00000001:3.0:1784948716.982619:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.982629:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948716.982632:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784948716.982633:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076985c84. 00000100:00000001:3.0:1784948716.982637:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948716.982641:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000069d2b91d. 00000100:00000001:3.0:1784948716.982642:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948716.982647:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948716.982649:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e5e96922. 00000020:00000010:0.0:1784948716.982653:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948716.982661:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948716.982664:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948716.982665:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948716.982668:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.982672:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948716.982675:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948716.982677:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.982692:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948716.982700:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948716.982702:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948716.982707:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 664 00000100:00000040:0.0:1784948716.982709:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948716.982712:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470020608 : -115544239531008 : ffff96e9c2a59c00) 00000100:00000040:0.0:1784948716.982717:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000026624f26 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:377/0 lens 224/0 e 0 to 0 dl 1784948727 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948716.982726:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948716.982729:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076985c84. 00000100:00000001:0.0:1784948716.982732:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948716.982734:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076985c84. 00000100:00100000:0.0:1784948716.982738:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000026624f26 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289648128:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948716.982744:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289648128 00000020:00000001:0.0:1784948716.982749:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948716.982753:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948716.982755:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948716.982758:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948716.982761:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948716.982765:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948716.982768:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948716.982771:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948716.982773:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948716.982776:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948716.982778:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948716.982784:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948716.982802:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948716.982807:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000081e7de16. 02000000:00000001:0.0:1784948716.982811:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.982814:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948716.982817:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948716.982820:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289648128 00010000:00000001:0.0:1784948716.982824:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948716.982830:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000026624f26 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:377/0 lens 224/224 e 0 to 0 dl 1784948727 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948716.982843:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948716.982846:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948716.982848:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=33 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948716.982852:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948716.982855:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948716.982857:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948716.982860:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948716.982861:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948716.982863:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948716.982866:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948716.982869:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 00000100:00000200:0.0:1784948716.982872:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289648128, offset 224 00000400:00000200:0.0:1784948716.982877:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948716.982885:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948716.982891:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3139:3139:240:4294967295] 192.168.202.137@tcp LPNI seq info [2046:2046:8:4294967295] 00000400:00000200:0.0:1784948716.982901:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948716.982906:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948716.982910:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d1b12746. 00000800:00000200:0.0:1784948716.982915:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948716.982946:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948716.982950:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d1b12746 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948716.982964:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948716.982967:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948716.982969:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948716.982971:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948716.982973:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948716.982977:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000026624f26 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:377/0 lens 224/224 e 0 to 0 dl 1784948727 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948716.982986:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000026624f26 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289648128:12345-192.168.202.137@tcp:400: Request processed in 259us (546us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948716.982994:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 664 00000100:00000040:0.0:1784948716.982997:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948716.983000:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948716.983001:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948716.983004:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e5e96922. 00000020:00000010:0.0:1784948716.983008:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948716.983011:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000069d2b91d. 00000020:00000040:0.0:1784948716.983015:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948716.983017:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948716.983121:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948716.983127:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d1b12746. 00000400:00000200:1.0:1784948716.983132:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948716.983139:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948716.983145:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000400:00000010:1.0:1784948716.983149:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000100:00000001:1.0:1784948716.983156:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948716.983159:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784948716.983169:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948716.983181:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948716.983185:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948716.983206:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948716.983214:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948716.983227:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434c00 00000400:00000200:1.0:1784948716.983236:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x807d [1] + 224 00000800:00000001:1.0:1784948716.983244:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948716.983250:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948716.983254:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948716.983259:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948716.983264:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948716.983270:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000095d61ef9 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948716.983297:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000095d61ef9 x1871651289648128/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948732 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948716.983345:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948716.983353:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948716.983357:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948717.024037:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305656363, at 4305656364 00000100:00000040:0.0:1784948717.024047:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305656363, at 4305656364 00000100:00000010:3.0:1784948717.024061:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000010:0.0:1784948717.024065:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:1.0:1784948717.024115:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305656363, at 4305656364 00000100:00000010:1.0:1784948717.024127:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000020:00000001:1.0:1784948717.812065:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948717.812068:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948717.812071:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948718.836046:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948718.836050:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948718.836053:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948719.860191:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948719.860197:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948719.860202:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1784948720.884253:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948720.884261:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948720.884266:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948721.908182:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948721.908188:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948721.908194:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784948722.100073:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948722.100086:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948722.100098:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000020:00000040:0.0:1784948722.100107:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948722.100115:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948722.100119:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948722.100127:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948722.100138:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b51ad697. 00000100:00000001:0.0:1784948722.100151:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948722.100164:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b3c18d2b x1871651289648256/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948722.100191:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948722.100196:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948722.100202:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_00_03+3] req@00000000b3c18d2b x1871651289648256/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948722.100252:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948722.100255:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948722.100289:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948722.100293:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948722.100298:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948722.100303:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009778919c. 00000020:00000040:0.0:1784948722.100307:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948722.100310:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948722.100312:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948722.100315:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948722.100322:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000086e4d21. 00000100:00000001:0.0:1784948722.100327:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948722.100336:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948722.100355:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948722.100361:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948722.100372:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009778919c] to pc [ptlrpcd_00_00+0] req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948722.100455:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948722.100466:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10999) 00000100:00000001:1.0:1784948722.100488:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948722.100495:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948722.100501:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948722.100510:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948722.100517:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948722.100519:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948722.100543:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948722.100546:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948722.100548:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.100552:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948722.100559:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948722.100563:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948722.100569:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948722.100575:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000009778919c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289648384:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948722.100604:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948722.100612:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948722.100615:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948722.100620:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948722.100630:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007465fe15. 02000000:00000001:1.0:1784948722.100637:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948722.100668:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000877cac20. 00000400:00000010:1.0:1784948722.100681:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000100:00000200:1.0:1784948722.100700:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289648384, portal 10 00000100:00000001:1.0:1784948722.100707:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948722.100712:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558294016 : -115544151257600 : ffff96e9c7e88e00) 00000100:00000040:1.0:1784948722.100727:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948722.100751:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948722.100763:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948722.100770:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000200:1.0:1784948722.100780:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289648384, offset 0 00000400:00000200:1.0:1784948722.100794:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948722.100818:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948722.100827:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948722.100833:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948722.100837:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948722.100842:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948722.100846:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3140:3140:240:4294967295] 192.168.202.137@tcp LPNI seq info [2047:2047:8:4294967295] 00000400:00000200:1.0:1784948722.100862:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948722.100871:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948722.100883:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000057c7ab4. 00000800:00000200:1.0:1784948722.100899:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948722.100964:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948722.100974:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000057c7ab4 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948722.100996:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948722.101004:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2028 usecs (3 jiffies) 00000100:00000001:1.0:1784948722.101013:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305661444 00000100:00000010:1.0:1784948722.101023:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000001:1.0:1784948722.101038:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784948722.101051:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305661444 00000100:00000010:1.0:1784948722.101061:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000052b5f963. 00000100:00000001:1.0:1784948722.101070:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948722.101079:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305661444 00000100:00000010:1.0:1784948722.101086:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:1.0:1784948722.101099:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948722.101108:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948722.101114:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305661444 00000100:00100000:3.0:1784948722.101118:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289648256 00000100:00000010:1.0:1784948722.101121:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000025674e46. 02000000:00000001:3.0:1784948722.101129:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948722.101133:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948722.101137:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948722.101141:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948722.101145:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.101150:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.101158:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948722.101158:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.101169:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948722.101174:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289648256 00000020:00000001:3.0:1784948722.101182:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948722.101183:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948722.101189:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000040:3.0:1784948722.101192:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784948722.101197:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948722.101202:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784948722.101204:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948722.101213:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948722.101213:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948722.101227:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948722.101227:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784948722.101241:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948722.101255:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948722.101299:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001f45750. 00000100:00000001:3.0:1784948722.101308:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948722.101317:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001f45750. 00000800:00000001:1.0:1784948722.101333:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948722.101334:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1784948722.101336:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000800:00000010:0.0:1784948722.101344:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000057c7ab4. 00000800:00000001:1.0:1784948722.101352:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948722.101354:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013785842. 00000400:00000200:0.0:1784948722.101355:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948722.101359:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948722.101369:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948722.101371:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948722.101372:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000400:00000200:0.0:1784948722.101376:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000400:00000010:0.0:1784948722.101384:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000400:00000200:1.0:1784948722.101387:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948722.101394:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1784948722.101398:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000200:0.0:1784948722.101405:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948722.101409:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434d00 00000100:00000001:3.0:1784948722.101410:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948722.101418:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948722.101427:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948722.101434:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 150000 00000100:00000001:0.0:1784948722.101435:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948722.101447:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948722.101448:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784948722.101449:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948722.101465:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948722.101468:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784948722.101469:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948722.101475:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948722.101477:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948722.101487:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948722.101497:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948722.101500:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948722.101506:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007fef3945. 00000100:00000001:3.0:1784948722.101509:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:1.0:1784948722.101512:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000007fef3945 x1871651289648384 msgsize 224 00000100:00000001:3.0:1784948722.101519:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948722.101528:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948722.101529:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948722.101536:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948722.101538:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 689 00000100:00000040:3.0:1784948722.101548:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948722.101551:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948722.101552:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948722.101554:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818140240 : -115538891411376 : ffff96eb016b7050) 00000100:00000001:0.0:1784948722.101559:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948722.101565:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948722.101572:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f3ed5d93 x1871651289648256/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:383/0 lens 224/0 e 0 to 0 dl 1784948733 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948722.101574:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948722.101598:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948722.101605:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001f45750. 00000100:00000001:1.0:1784948722.101610:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948722.101612:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1784948722.101615:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289648384 02000000:00000001:1.0:1784948722.101624:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000010:3.0:1784948722.101624:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001f45750. 00000100:00000001:1.0:1784948722.101629:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948722.101635:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948722.101638:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948722.101642:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289648384 00000100:00100000:3.0:1784948722.101646:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f3ed5d93 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289648256:12345-0@lo:400: 00000020:00000001:1.0:1784948722.101647:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948722.101651:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948722.101656:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948722.101664:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:3.0:1784948722.101665:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289648256 00000020:00000001:1.0:1784948722.101669:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948722.101674:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784948722.101676:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784948722.101678:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948722.101688:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000030655a79. 00000020:00000001:3.0:1784948722.101689:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784948722.101692:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948722.101697:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948722.101698:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000030655a79. 00000020:00000001:3.0:1784948722.101707:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000010:1.0:1784948722.101709:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003b287909. 00000020:00000001:3.0:1784948722.101712:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000010:1.0:1784948722.101717:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004f0879c9. 00000020:00000001:3.0:1784948722.101721:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000010:1.0:1784948722.101724:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000020:00000001:3.0:1784948722.101727:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948722.101734:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784948722.101737:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:3.0:1784948722.101738:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948722.101745:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000001:3.0:1784948722.101748:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.101749:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948722.101755:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948722.101757:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948722.101764:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948722.101765:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948722.101773:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:3.0:1784948722.101774:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948722.101777:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948722.101785:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 02000000:00000001:3.0:1784948722.101793:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948722.101798:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.101799:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948722.101810:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948722.101814:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289648256 00010000:00000001:3.0:1784948722.101818:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784948722.101821:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948722.101825:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000200:3.0:1784948722.101829:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f3ed5d93 x1871651289648256/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:383/0 lens 224/224 e 0 to 0 dl 1784948733 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948722.101839:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 665 00000100:00000040:1.0:1784948722.101849:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00010000:00000001:3.0:1784948722.101852:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784948722.101854:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451660416 : -115544257891200 : ffff96e9c18d7480) 00010000:00000001:3.0:1784948722.101857:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948722.101863:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=44 v=5 (1 1 1 1) 00000100:00000040:1.0:1784948722.101866:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000007fef3945 x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:383/0 lens 224/0 e 0 to 0 dl 1784948733 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948722.101875:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948722.101884:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948722.101886:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948722.101892:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000010:1.0:1784948722.101894:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000030655a79. 02000000:00000001:3.0:1784948722.101898:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784948722.101902:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948722.101939:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948722.101942:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000030655a79. 00000100:00000001:3.0:1784948722.101948:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1784948722.101958:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000007fef3945 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289648384:12345-192.168.202.137@tcp:400: 00000100:00000040:3.0:1784948722.101959:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000200:1.0:1784948722.101968:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289648384 00000400:00000010:3.0:1784948722.101970:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000020:00000001:1.0:1784948722.101972:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000200:3.0:1784948722.101981:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289648256, offset 224 00000020:00000001:1.0:1784948722.101982:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948722.101986:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948722.101991:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:3.0:1784948722.101997:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:1.0:1784948722.101998:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948722.102006:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948722.102010:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948722.102017:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:3.0:1784948722.102017:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:1.0:1784948722.102020:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948722.102025:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948722.102028:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:3.0:1784948722.102034:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434c80 00000100:00000001:1.0:1784948722.102036:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948722.102041:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:3.0:1784948722.102047:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8095 [1] + 224 02000000:00000010:1.0:1784948722.102050:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f147cc65. 02000000:00000001:1.0:1784948722.102056:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948722.102056:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948722.102060:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948722.102067:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:1.0:1784948722.102069:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948722.102074:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289648384 00000100:00000001:3.0:1784948722.102076:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:1.0:1784948722.102078:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000200:3.0:1784948722.102085:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b3c18d2b x1871651289648256/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784948722.102088:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000007fef3945 x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:383/0 lens 224/224 e 0 to 0 dl 1784948733 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948722.102141:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948722.102146:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948722.102148:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b3c18d2b x1871651289648256/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:1.0:1784948722.102150:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=39 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948722.102163:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948722.102171:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948722.102178:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948722.102184:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948722.102186:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.102189:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948722.102193:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948722.102194:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000010:1.0:1784948722.102199:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000100:00000200:1.0:1784948722.102209:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289648384, offset 224 00000400:00000200:3.0:1784948722.102210:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:3.0:1784948722.102216:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:3.0:1784948722.102224:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948722.102229:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948722.102231:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948722.102239:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948722.102252:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:1.0:1784948722.102274:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00010000:00000001:3.0:1784948722.102283:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948722.102292:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948722.102296:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3141:3141:240:4294967295] 192.168.202.137@tcp LPNI seq info [2048:2048:8:4294967295] 00000020:00000001:3.0:1784948722.102303:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948722.102319:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f3ed5d93 x1871651289648256/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:383/0 lens 224/224 e 0 to 0 dl 1784948733 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948722.102341:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948722.102370:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948722.102381:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000820f02b3. 00000800:00000200:1.0:1784948722.102404:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:3.0:1784948722.102405:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f3ed5d93 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289648256:12345-0@lo:400: Request processed in 795us (1488us total) trans 0 rc 0/0 00000800:00000200:1.0:1784948722.102425:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948722.102433:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000820f02b3 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:3.0:1784948722.102441:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 689 00000100:00000040:3.0:1784948722.102452:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948722.102481:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948722.102483:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948722.102490:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:1.0:1784948722.102498:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000020:00000010:3.0:1784948722.102503:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013785842. 00010000:00000001:1.0:1784948722.102506:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948722.102513:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948722.102519:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000010:3.0:1784948722.102519:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000100:00000040:1.0:1784948722.102532:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007fef3945 x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:383/0 lens 224/224 e 0 to 0 dl 1784948733 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948722.102535:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000020:00000040:3.0:1784948722.102552:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:3.0:1784948722.102568:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948722.102575:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 02000000:00000001:3.0:1784948722.102587:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948722.102597:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948722.102606:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000007fef3945 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289648384:12345-192.168.202.137@tcp:400: Request processed in 710us (1083us total) trans 0 rc 0/0 00000100:00000200:3.0:1784948722.102617:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:1.0:1784948722.102628:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 665 00000400:00000010:3.0:1784948722.102634:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000100:00000040:1.0:1784948722.102640:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948722.102650:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:3.0:1784948722.102651:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000042b6a6f7. 00000100:00000001:1.0:1784948722.102674:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948722.102683:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004f0879c9. 00000800:00000200:0.0:1784948722.102684:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:1.0:1784948722.102695:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000800:00000010:0.0:1784948722.102697:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000820f02b3. 00000020:00000010:1.0:1784948722.102705:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000003b287909. 00000400:00000200:0.0:1784948722.102714:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1784948722.102717:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948722.102727:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948722.102732:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948722.102738:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948722.102753:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948722.102755:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000800:00000001:3.0:1784948722.102764:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:0.0:1784948722.102765:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000800:00000001:3.0:1784948722.102778:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948722.102779:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948722.102788:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948722.102804:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948722.102831:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434d00 00000400:00000200:3.0:1784948722.102854:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x809d [1] + 224 00000800:00000001:3.0:1784948722.102866:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948722.102875:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948722.102879:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948722.102886:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948722.102893:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948722.102899:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948722.102916:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948722.102979:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948722.102991:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948722.102992:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948722.102997:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:3.0:1784948722.103001:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:1.0:1784948722.103004:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948722.103010:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948722.103013:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000877cac20. 00000400:00000200:1.0:1784948722.103018:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:1.0:1784948722.103021:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000100:00000001:1.0:1784948722.103024:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948722.103031:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948722.103043:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948722.103051:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948722.103053:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948722.103055:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948722.103057:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948722.103059:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948722.103061:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948722.103064:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948722.103069:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=39 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948722.103073:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=39 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948722.103076:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948722.103077:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948722.103080:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948722.103083:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948722.103091:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.103095:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948722.103097:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948722.103099:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948722.103101:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948722.103105:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948722.103112:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948722.103114:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948722.103116:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948722.103119:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948722.103126:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000009778919c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289648384:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948722.103131:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948722.103133:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948722.103136:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000009778919c x1871651289648384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948722.103143:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948722.103145:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948722.103147:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007465fe15. 02000000:00000001:1.0:1784948722.103150:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948722.103151:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948722.103153:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948722.103155:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948722.103156:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000086e4d21. 02000000:00000001:1.0:1784948722.103159:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948722.103161:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948722.103162:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000009778919c. 00000100:00000001:1.0:1784948722.103166:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948722.103168:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948722.103170:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948722.103173:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948722.103175:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948722.103177:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948722.103179:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948722.143984:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305661483, at 4305661484 00000100:00000010:0.0:1784948722.143999:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000040:1.0:1784948722.144022:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305661483, at 4305661484 00000100:00000010:1.0:1784948722.144035:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000052b5f963. 00000100:00000040:3.0:1784948722.144044:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305661483, at 4305661484 00000100:00000010:3.0:1784948722.144061:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000025674e46. 00000800:00000200:0.0:1784948722.292055:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:0.0:1784948722.292066:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000800:00000200:0.0:1784948722.292071:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.0:1784948722.292329:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:3.0:1784948722.932198:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948722.932205:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948722.932209:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948723.124052:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948723.124065:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948723.124068:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948723.124074:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948723.124079:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948723.124082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948723.124085:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948723.956187:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948723.956194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948723.956198:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948724.148034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948724.148051:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948724.148054:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948724.148060:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948724.148063:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948724.148067:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948724.148071:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948724.340058:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000800:00000010:0.0:1784948724.340066:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005e0a2c1e. 00000400:00000200:0.0:1784948724.340079:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948724.340088:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948724.340099:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 10997 health :900 00000400:00000200:0.0:1784948724.340106:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948724.340111:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:0.0:1784948724.340116:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000800:00000100:0.0:1784948724.340150:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000800:00000010:0.0:1784948724.340153:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000faf98f5b. 00000400:00000200:0.0:1784948724.340157:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948724.340163:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948724.340167:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948724.340172:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:0.0:1784948724.340177:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000200:3.0:1784948724.340192:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948724.340205:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:3.0:1784948724.340215:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948724.340219:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948724.340224:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f43fd4d9 00000400:00000010:3.0:1784948724.340229:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f43fd4d9. 00000400:00000200:3.0:1784948724.340317:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948724.340327:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:3.0:1784948724.340335:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000075318cf7. 00000400:00000200:3.0:1784948724.340345:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:3.0:1784948724.340350:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:3.0:1784948724.340357:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:3.0:1784948724.340364:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:3.0:1784948724.340369:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948724.340370:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948724.340372:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000236111b7 00000400:00000010:3.0:1784948724.340375:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000236111b7. 00000400:00000200:3.0:1784948724.340379:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948724.340382:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:3.0:1784948724.340385:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000244a9cae. 00000400:00000200:3.0:1784948724.340390:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:3.0:1784948724.340393:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:3.0:1784948724.340396:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000020:00000001:3.0:1784948724.980147:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948724.980153:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948724.980156:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948725.172141:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948725.172156:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948725.172159:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948725.172165:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948725.172168:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948725.172172:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948725.172175:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948725.364009:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000800:00000010:0.0:1784948725.364016:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001adbd2d9. 00000400:00000200:0.0:1784948725.364028:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948725.364036:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948725.364048:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948725.364055:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:0.0:1784948725.364063:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:3.0:1784948725.364076:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948725.364099:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000800:00000100:0.0:1784948725.364101:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000010:0.0:1784948725.364110:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000eafb2cdc. 00000400:00000200:1.0:1784948725.364112:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000001:3.0:1784948725.364117:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:0.0:1784948725.364118:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:1.0:1784948725.364125:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000048fd39a3. 00000400:00000001:3.0:1784948725.364127:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948725.364136:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948725.364138:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e7a7cb2 00000400:00000010:1.0:1784948725.364142:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007465fe15. 00000400:00000200:0.0:1784948725.364146:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 10998 health :600 00000400:00000010:3.0:1784948725.364148:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002e7a7cb2. 00000400:00000010:1.0:1784948725.364152:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000200:3.0:1784948725.364164:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948725.364165:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:1.0:1784948725.364170:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003b463cff. 00000400:00000200:0.0:1784948725.364179:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:3.0:1784948725.364188:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:1.0:1784948725.364189:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948725.364202:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef171f31. 00000400:00000200:3.0:1784948725.364216:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000200:0.0:1784948725.364219:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:3.0:1784948725.364262:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:1.0:1784948725.364270:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948725.364283:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948725.364288:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3142:3142:243:4294967295] 192.168.202.137@tcp LPNI seq info [2049:2049:8:4294967295] 00000400:00000200:1.0:1784948725.364297:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948725.364307:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948725.364315:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948725.364322:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009ff40a64. 00000400:00000200:3.0:1784948725.364325:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000800:00000200:1.0:1784948725.364338:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948725.364345:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:3.0:1784948725.364355:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1784948725.364357:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:3.0:1784948725.364361:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1784948725.364365:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009ff40a64 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784948725.364368:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00000400:00000010:3.0:1784948725.364375:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000400:00000200:3.0:1784948725.364382:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948725.364391:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:3.0:1784948725.364398:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001df3b40e. 00000400:00000200:3.0:1784948725.364408:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:3.0:1784948725.364416:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:3.0:1784948725.364425:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000800:00000200:0.0:1784948725.364566:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948725.364572:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009ff40a64. 00000400:00000200:0.0:1784948725.364577:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948725.364584:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948725.364589:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948725.364594:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784948725.364607:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948725.364621:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948725.364625:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948725.364632:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948725.364640:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948725.364654:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948725.364665:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948725.364676:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948725.364682:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3143:3143:244:4294967295] 192.168.202.137@tcp LPNI seq info [2050:2050:8:4294967295] 00000400:00000200:0.0:1784948725.364693:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948725.364700:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948725.364704:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000eafb2cdc. 00000800:00000200:0.0:1784948725.364709:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948725.364715:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948725.364719:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000eafb2cdc type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784948725.364817:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948725.364828:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784948725.364830:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948725.364835:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000eafb2cdc. 00000800:00000001:1.0:1784948725.364841:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948725.364843:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948725.364852:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784948725.364856:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948725.364865:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948725.364873:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948725.364890:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784948725.364945:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x80b1 00000800:00000001:1.0:1784948725.364952:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948725.364965:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948725.364968:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948725.364972:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784948725.364979:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:1.0:1784948725.364982:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000400:00000200:1.0:1784948725.364985:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784948725.364989:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000048fd39a3. 00000400:00000010:1.0:1784948725.364993:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007465fe15. 00000800:00000001:1.0:1784948725.365001:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948725.365003:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948726.004086:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948726.004089:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948726.004092:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948726.196087:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948726.196133:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948726.196135:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948726.196142:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948726.196146:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948726.196151:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948726.196154:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948726.388028:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000800:00000010:0.0:1784948726.388037:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000868cf222. 00000400:00000200:0.0:1784948726.388050:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948726.388060:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948726.388074:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 10999 health :900 00000400:00000200:0.0:1784948726.388081:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948726.388087:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:0.0:1784948726.388096:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:3.0:1784948726.388166:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948726.388179:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:3.0:1784948726.388187:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948726.388190:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948726.388195:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:3.0:1784948726.388200:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000400:00000200:3.0:1784948726.388208:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948726.388215:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:3.0:1784948726.388246:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005cc3cd5d. 00000400:00000200:3.0:1784948726.388273:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:3.0:1784948726.388277:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:3.0:1784948726.388282:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000020:00000001:3.0:1784948727.028124:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948727.028130:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948727.028135:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948727.220019:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948727.220038:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948727.220041:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948727.220048:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948727.220052:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948727.220057:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948727.220061:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948727.220119:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948727.220126:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948727.220138:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000dc33132b. 00000020:00000040:1.0:1784948727.220147:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948727.220154:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948727.220158:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948727.220167:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948727.220177:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005a1c05a3. 00000100:00000001:1.0:1784948727.220188:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948727.220205:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948727.220248:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948727.220252:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948727.220263:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc33132b] to pc [ptlrpcd_00_01+1] req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948727.220314:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948727.220319:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948727.220325:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948727.220329:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948727.220334:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948727.220340:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000941b125. 00000100:00000001:0.0:1784948727.220345:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948727.220345:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948727.220355:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948727.220356:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948727.220362:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784948727.220363:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948727.220371:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948727.220377:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784948727.220383:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000009ff40a64. 00000100:00000001:0.0:1784948727.220395:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948727.220401:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948727.220405:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948727.220412:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784948727.220421:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000000941b125 x1871651289648640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1784948727.220422:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948727.220436:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948727.220453:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000dc33132b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289648512:0@lo:400: 00000100:00000001:0.0:1784948727.220473:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784948727.220476:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948727.220480:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948727.220485:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1784948727.220487:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000941b125] to pc [ptlrpcd_00_02+2] req@000000000941b125 x1871651289648640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948727.220494:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948727.220506:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948727.220517:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948727.220521:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948727.220530:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948727.220531:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c8dbdfb2. 00000100:00000040:1.0:1784948727.220545:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11004) 00000400:00000010:0.0:1784948727.220547:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000100:00000200:0.0:1784948727.220559:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289648512, portal 25 00000100:00000001:0.0:1784948727.220565:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948727.220568:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451662208 : -115544257889408 : ffff96e9c18d7b80) 00000100:00000040:0.0:1784948727.220580:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948727.220589:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948727.220598:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.220602:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948727.220618:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948727.220629:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000100:00000200:0.0:1784948727.220636:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289648512, offset 0 00000400:00000200:0.0:1784948727.220656:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948727.220686:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948727.220713:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434d80 00000400:00000200:0.0:1784948727.220735:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7725 [2] + 0 00000400:00000010:0.0:1784948727.220746:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000003793f8ab. 00000400:00000200:0.0:1784948727.220756:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f45face5 00000400:00000200:0.0:1784948727.220770:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948727.220787:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948727.220801:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f45face5 00000400:00000010:0.0:1784948727.220810:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f45face5. 00000100:00000001:0.0:1784948727.220827:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948727.220834:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948727.220855:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f24717d4 x1871651289648512 msgsize 224 00000100:00100000:0.0:1784948727.220884:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948727.220889:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948727.220943:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948727.220978:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948727.220990:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289648512 02000000:00000001:3.0:1784948727.221013:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948727.221021:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:0.0:1784948727.221022:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:0.0:1784948727.221031:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000100:00000001:3.0:1784948727.221033:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.221042:0:337120:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784948727.221047:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948727.221053:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948727.221059:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289648512 00000020:00000001:3.0:1784948727.221070:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948727.221080:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948727.221090:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1784948727.221091:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948727.221101:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000040:0.0:1784948727.221102:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948727.221115:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784948727.221130:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948727.221148:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948727.221149:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.221161:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784948727.221163:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d92bf133. 00000100:00000001:0.0:1784948727.221171:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948727.221173:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948727.221182:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305666603 00000020:00000010:3.0:1784948727.221185:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d92bf133. 00000100:00000001:0.0:1784948727.221197:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305666564 00000020:00000010:3.0:1784948727.221208:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000100:00000001:0.0:1784948727.221209:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305666603 00000100:00000001:0.0:1784948727.221253:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305666564 00000020:00000010:3.0:1784948727.221269:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013785842. 00000100:00000001:0.0:1784948727.221272:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305666603 00000020:00000010:3.0:1784948727.221284:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000040:3.0:1784948727.221306:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948727.221307:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305666564 00000100:00000001:3.0:1784948727.221318:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948727.221326:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305666603 00000100:00000001:3.0:1784948727.221327:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948727.221337:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.221340:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305666564 00000100:00000001:0.0:1784948727.221357:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948727.221360:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.221368:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.221375:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.221382:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948727.221390:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948727.221398:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948727.221407:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948727.221417:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948727.221420:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948727.221422:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948727.221430:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948727.221432:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.221442:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948727.221448:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 690 00000100:00000040:3.0:1784948727.221458:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948727.221463:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457130576 : -115544252421040 : ffff96e9c1e0ec50) 00000100:00000040:3.0:1784948727.221475:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f24717d4 x1871651289648512/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:388/0 lens 224/0 e 0 to 0 dl 1784948738 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948727.221505:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:1.0:1784948727.221511:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1784948727.221514:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d92bf133. 00000800:00000010:1.0:1784948727.221521:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009839174d. 00000100:00000001:3.0:1784948727.221524:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948727.221534:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784948727.221535:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d92bf133. 00000800:00000001:0.0:1784948727.221544:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948727.221550:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:3.0:1784948727.221554:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f24717d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289648512:12345-0@lo:400: 00000400:00000200:1.0:1784948727.221562:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000800:00000001:0.0:1784948727.221565:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948727.221571:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00000200:3.0:1784948727.221578:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289648512 00000800:00000001:0.0:1784948727.221579:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948727.221583:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:0.0:1784948727.221589:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948727.221591:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000200:1.0:1784948727.221596:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000941b125 x1871651289648640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948727.221604:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948727.221614:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948727.221617:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:3.0:1784948727.221626:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948727.221633:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:0.0:1784948727.221644:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434e00 00000020:00000001:3.0:1784948727.221644:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948727.221654:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948727.221655:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784948727.221665:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 150224 00000020:00000001:3.0:1784948727.221667:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784948727.221675:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000941b125 x1871651289648640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948743 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948727.221677:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:0.0:1784948727.221679:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948727.221689:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948727.221693:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948727.221695:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948727.221698:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948727.221700:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948727.221703:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948727.221709:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:0.0:1784948727.221714:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784948727.221733:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784948727.221738:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948727.221744:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000010:3.0:1784948727.221745:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fd4fe1c8. 00000100:00000010:0.0:1784948727.221754:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e248b6f7. 02000000:00000001:3.0:1784948727.221756:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948727.221767:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000e248b6f7 x1871651289648640 msgsize 224 00000100:00000001:3.0:1784948727.221768:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948727.221781:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948727.221795:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000040:3.0:1784948727.221795:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289648512 00010000:00000001:3.0:1784948727.221807:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948727.221818:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f24717d4 x1871651289648512/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:388/0 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948727.221824:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000001:3.0:1784948727.221841:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000800:00000001:0.0:1784948727.221842:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:3.0:1784948727.221850:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948727.221852:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784948727.221862:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=49 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948727.221874:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948727.221881:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948727.221886:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948727.221894:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948727.221902:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948727.221978:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948727.221995:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948727.222007:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000100:00000200:3.0:1784948727.222022:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289648512, offset 224 00000400:00000200:3.0:1784948727.222054:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948727.222078:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948727.222103:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434d80 00000400:00000200:3.0:1784948727.222129:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x80b5 [1] + 224 00000400:00000200:3.0:1784948727.222152:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948727.222168:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948727.222181:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948727.222188:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948727.222196:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:3.0:1784948727.222198:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948727.222240:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.222259:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948727.222283:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948727.222350:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1784948727.222370:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:3.0:1784948727.222381:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:0.0:1784948727.222387:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948727.222395:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948727.222396:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948727.222406:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:0.0:1784948727.222409:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948727.222420:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784948727.222420:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948727.222430:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c8dbdfb2. 00000100:00000040:3.0:1784948727.222438:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:0.0:1784948727.222444:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00010000:00000001:3.0:1784948727.222454:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:0.0:1784948727.222455:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000020:00000001:3.0:1784948727.222464:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.222468:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784948727.222474:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:0.0:1784948727.222483:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948727.222492:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f24717d4 x1871651289648512/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:388/0 lens 224/224 e 0 to 0 dl 1784948738 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948727.222553:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948727.222576:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f24717d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289648512:12345-0@lo:400: Request processed in 1061us (1710us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948727.222598:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948727.222607:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948727.222617:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00100000:3.0:1784948727.222621:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 690 02000000:00000001:0.0:1784948727.222625:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948727.222633:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784948727.222636:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948727.222641:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948727.222646:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:0.0:1784948727.222652:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948727.222655:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00001000:0.0:1784948727.222664:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=49 v=5 (5 5 5 5) 00000020:00000010:3.0:1784948727.222670:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013785842. 00000100:00001000:0.0:1784948727.222680:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=49 v=5 (1 1 1 1) 00000020:00000010:3.0:1784948727.222685:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000020:00000010:3.0:1784948727.222699:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000100:00000001:0.0:1784948727.222700:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948727.222706:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948727.222723:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000040:3.0:1784948727.222728:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000001:0.0:1784948727.222729:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.222742:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948727.222746:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948727.222755:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000fd4fe1c8. 00000100:00000040:0.0:1784948727.222758:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948727.222768:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948727.222777:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948727.222796:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948727.222809:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784948727.222814:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c6713e8b. 00000100:00000001:0.0:1784948727.222821:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948727.222830:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948727.222832:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000092ad473a. 00000100:00000040:0.0:1784948727.222846:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948727.222868:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000dc33132b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289648512:0@lo:400: 00000100:00000001:0.0:1784948727.222889:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948727.222900:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948727.222966:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000dc33132b x1871651289648512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948727.222993:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948727.223000:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948727.223005:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948727.223016:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948727.223021:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948727.223027:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948727.223034:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948727.223041:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005a1c05a3. 02000000:00000001:0.0:1784948727.223059:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948727.223064:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948727.223069:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000dc33132b. 00000800:00000200:1.0:1784948727.223073:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948727.223077:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948727.223082:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784948727.223083:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000925a7f5a. 00000100:00000001:0.0:1784948727.223088:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948727.223096:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948727.223098:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948727.223104:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948727.223111:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948727.223113:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948727.223117:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948727.223121:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000800:00000001:3.0:1784948727.223128:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784948727.223129:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000100:00000001:1.0:1784948727.223141:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948727.223145:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948727.223150:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948727.223158:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948727.223166:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948727.223182:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948727.223199:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434e00 00000400:00000200:3.0:1784948727.223209:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x80bd [1] + 224 00000800:00000001:3.0:1784948727.223246:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948727.223283:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948727.223289:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948727.223295:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948727.223303:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948727.223309:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000941b125 x1871651289648640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948727.223330:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000941b125 x1871651289648640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948727.223364:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948727.223374:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948727.223380:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948727.263980:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305666603, at 4305666604 00000100:00000010:0.0:1784948727.263999:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000100:00000040:3.0:1784948727.264029:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305666603, at 4305666604 00000100:00000040:1.0:1784948727.264034:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305666603, at 4305666604 00000100:00000010:3.0:1784948727.264049:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000445487c0. 00000100:00000010:1.0:1784948727.264059:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008a834f68. 00000020:00000001:1.0:1784948728.052155:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948728.052163:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948728.052168:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948728.244066:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948728.244084:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948728.244087:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948728.244093:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948728.244097:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948728.244101:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948728.244104:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948729.076230:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948729.076236:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948729.076258:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948729.267997:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948729.268015:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948729.268019:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948729.268026:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948729.268031:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948729.268035:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948729.268039:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948729.460098:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000800:00000010:0.0:1784948729.460106:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005d1384a4. 00000400:00000200:0.0:1784948729.460117:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948729.460135:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948729.460143:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11002 health :900 00000400:00000200:0.0:1784948729.460150:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948729.460155:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:0.0:1784948729.460160:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:3.0:1784948729.460259:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948729.460270:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:3.0:1784948729.460278:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948729.460281:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948729.460284:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:3.0:1784948729.460287:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000400:00000200:3.0:1784948729.460292:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948729.460297:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:3.0:1784948729.460302:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000400:00000200:3.0:1784948729.460310:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:3.0:1784948729.460313:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:3.0:1784948729.460317:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000020:00000001:1.0:1784948730.100268:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948730.100276:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948730.100283:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948730.292035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948730.292048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948730.292050:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948730.292055:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948730.292058:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948730.292061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948730.292063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948730.484076:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:0.0:1784948730.484079:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.7@tcp 00000800:00000010:0.0:1784948730.484086:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e471e641. 00000400:00000010:1.0:1784948730.484092:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000048fd39a3. 00000400:00000200:0.0:1784948730.484101:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:1.0:1784948730.484103:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007465fe15. 00000400:00000200:0.0:1784948730.484110:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784948730.484112:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000200:0.0:1784948730.484119:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11003 health :800 00000400:00000010:1.0:1784948730.484125:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000091a157d3. 00000400:00000200:0.0:1784948730.484127:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948730.484133:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948730.484134:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:0.0:1784948730.484142:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:1.0:1784948730.484186:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948730.484225:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948730.484232:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3146:3146:247:4294967295] 192.168.202.137@tcp LPNI seq info [2053:2053:8:4294967295] 00000400:00000200:1.0:1784948730.484244:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948730.484258:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000400:00000200:3.0:1784948730.484271:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:1.0:1784948730.484273:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948730.484284:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000800:00000010:1.0:1784948730.484286:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000820f02b3. 00000400:00000001:3.0:1784948730.484295:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948730.484301:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1784948730.484303:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948730.484306:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553902b5 00000400:00000010:3.0:1784948730.484311:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000553902b5. 00000400:00000200:3.0:1784948730.484317:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000200:1.0:1784948730.484323:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:3.0:1784948730.484328:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000800:00000200:1.0:1784948730.484335:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000820f02b3 type 2, nob 96 niov 1 nkiov 0 00000400:00000010:3.0:1784948730.484338:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e7299d92. 00000400:00000200:3.0:1784948730.484350:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-110 00000400:00000200:3.0:1784948730.484353:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -110 00000400:00000200:3.0:1784948730.484358:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000800:00000200:0.0:1784948730.484549:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948730.484557:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000820f02b3. 00000400:00000200:0.0:1784948730.484564:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948730.484573:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784948730.484579:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948730.484580:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948730.484585:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948730.484596:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948730.484601:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948730.484609:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948730.484621:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948730.484639:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948730.484652:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948730.484668:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948730.484674:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3147:3147:247:4294967295] 192.168.202.137@tcp LPNI seq info [2054:2054:8:4294967295] 00000400:00000200:1.0:1784948730.484684:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948730.484690:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948730.484695:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000057c7ab4. 00000800:00000200:1.0:1784948730.484700:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948730.484706:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948730.484709:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000057c7ab4 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784948730.484823:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948730.484832:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784948730.484836:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784948730.484842:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000057c7ab4. 00000800:00000001:3.0:1784948730.484846:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948730.484851:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948730.484861:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948730.484864:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948730.484870:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948730.484882:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948730.484897:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948730.484967:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x80d1 00000800:00000001:3.0:1784948730.484977:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948730.484994:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948730.485000:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948730.485007:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948730.485015:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:3.0:1784948730.485019:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000400:00000200:3.0:1784948730.485024:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948730.485029:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000048fd39a3. 00000400:00000010:3.0:1784948730.485035:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007465fe15. 00000800:00000001:3.0:1784948730.485047:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948730.485052:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948731.124291:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948731.124300:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948731.124306:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948731.316010:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948731.316031:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948731.316035:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948731.316044:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948731.316050:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948731.316056:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948731.316060:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948731.508032:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.6@tcp 00000800:00000010:0.0:1784948731.508040:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c6406cdb. 00000400:00000200:0.0:1784948731.508054:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948731.508064:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948731.508073:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11004 health :900 00000400:00000200:0.0:1784948731.508080:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948731.508084:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:0.0:1784948731.508089:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000800:00000100:0.0:1784948731.508121:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.11@tcp 00000800:00000010:0.0:1784948731.508125:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c70e9e66. 00000400:00000200:0.0:1784948731.508128:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948731.508134:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948731.508139:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948731.508143:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:0.0:1784948731.508148:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:3.0:1784948731.508171:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948731.508243:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000400:00000001:3.0:1784948731.508254:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948731.508258:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948731.508262:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000400:00000010:3.0:1784948731.508267:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000400:00000200:3.0:1784948731.508275:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948731.508281:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:3.0:1784948731.508288:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e120de1c. 00000400:00000200:3.0:1784948731.508294:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-110 00000400:00000200:3.0:1784948731.508298:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -110 00000400:00000200:3.0:1784948731.508303:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:3.0:1784948731.508308:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:3.0:1784948731.508311:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948731.508312:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948731.508314:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:3.0:1784948731.508316:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000400:00000200:3.0:1784948731.508318:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948731.508321:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:3.0:1784948731.508324:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000013f68999. 00000400:00000200:3.0:1784948731.508333:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:3.0:1784948731.508335:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:3.0:1784948731.508338:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000020:00000001:1.0:1784948732.148271:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948732.148279:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948732.148285:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948732.340003:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948732.340018:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948732.340021:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948732.340029:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948732.340034:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948732.340038:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948732.340040:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948732.340152:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948732.340157:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948732.340160:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948732.340163:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948732.340165:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948732.340186:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000098475f18 x1871651289648768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948732.340208:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948732.340213:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948732.340222:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948732.340227:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948732.340235:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948732.340246:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000098475f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289648768:0@lo:400: 00000100:00000001:1.0:1784948732.340253:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948732.340261:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948732.340266:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948732.340274:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948732.340281:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000003b287909. 02000000:00000001:1.0:1784948732.340284:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948732.340290:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000877cac20. 00000100:00000001:0.0:1784948732.340294:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784948732.340297:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000100:00000001:0.0:1784948732.340302:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948732.340306:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289648768, portal 25 00000100:00000001:1.0:1784948732.340310:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948732.340314:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470016128 : -115544239535488 : ffff96e9c2a58a80) 00000100:00000040:1.0:1784948732.340325:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000098475f18 x1871651289648768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948732.340340:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948732.340346:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948732.340351:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000200:1.0:1784948732.340357:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289648768, offset 0 00000400:00000200:1.0:1784948732.340370:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948732.340385:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784948732.340404:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434e80 00000400:00000200:1.0:1784948732.340415:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x773d [2] + 0 00000400:00000010:1.0:1784948732.340422:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008c614621. 00000400:00000200:1.0:1784948732.340430:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000005616ec32 00000400:00000200:1.0:1784948732.340442:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948732.340453:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948732.340468:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005616ec32 00000400:00000010:1.0:1784948732.340473:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000005616ec32. 00000100:00000001:1.0:1784948732.340481:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948732.340484:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784948732.340489:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000028574ea9 x1871651289648768 msgsize 224 00000100:00100000:1.0:1784948732.340500:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784948732.340504:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948732.340516:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948732.340531:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:1.0:1784948732.340535:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:1.0:1784948732.340541:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784948732.340549:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000098475f18 x1871651289648768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948732.340562:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948732.340569:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000098475f18 x1871651289648768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948748 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948732.340570:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948732.340577:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289648768 00000100:00000001:1.0:1784948732.340581:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948732.340583:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948732.340585:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948732.340589:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948732.340591:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.340595:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948732.340597:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305671722 02000000:00000001:3.0:1784948732.340604:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948732.340611:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289648768 00000020:00000001:3.0:1784948732.340616:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948732.340619:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784948732.340624:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948732.340630:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948732.340634:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305671683 00000020:00000001:3.0:1784948732.340636:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948732.340643:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305671722 00000020:00000001:3.0:1784948732.340644:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948732.340655:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948732.340659:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305671683 00000020:00000010:3.0:1784948732.340662:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001f45750. 00000100:00000001:1.0:1784948732.340666:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305671722 00000100:00000001:3.0:1784948732.340669:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948732.340675:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001f45750. 00000100:00000001:1.0:1784948732.340682:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305671683 00000020:00000010:3.0:1784948732.340685:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000100:00000001:1.0:1784948732.340688:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305671722 00000020:00000010:3.0:1784948732.340696:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013785842. 00000020:00000010:3.0:1784948732.340706:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:1.0:1784948732.340708:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305671683 00000100:00000040:3.0:1784948732.340721:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948732.340722:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948732.340728:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.340729:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948732.340732:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.340733:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948732.340738:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948732.340739:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948732.340744:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948732.340749:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:0.0:1784948732.340750:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948732.340751:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948732.340756:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948732.340761:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948732.340765:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.340770:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948732.340771:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e6a3ade8. 00000100:00000001:1.0:1784948732.340772:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948732.340781:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948732.340791:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948732.340792:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784948732.340797:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000100:00000001:3.0:1784948732.340797:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:0.0:1784948732.340803:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000800:00000001:1.0:1784948732.340806:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:3.0:1784948732.340810:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 691 00000100:00000001:0.0:1784948732.340811:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:1.0:1784948732.340819:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948732.340819:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000200:0.0:1784948732.340822:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b8226094 x1871651289648896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948732.340827:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457133648 : -115544252417968 : ffff96e9c1e0f850) 00000800:00000001:1.0:1784948732.340828:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948732.340833:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948732.340841:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000028574ea9 x1871651289648768/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:393/0 lens 224/0 e 0 to 0 dl 1784948743 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948732.340849:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948732.340862:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948732.340866:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6434f00 00000100:00000001:3.0:1784948732.340870:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:0.0:1784948732.340871:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b8226094 x1871651289648896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948748 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948732.340878:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001f45750. 00000400:00000200:1.0:1784948732.340881:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 150448 00000100:00000001:3.0:1784948732.340887:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948732.340895:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948732.340896:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784948732.340896:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001f45750. 00000100:00000001:0.0:1784948732.340898:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784948732.340931:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948732.340935:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:3.0:1784948732.340940:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000028574ea9 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289648768:12345-0@lo:400: 00000400:00000200:1.0:1784948732.340941:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948732.340951:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948732.340951:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289648768 00000100:00000200:1.0:1784948732.340954:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948732.340960:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000600c5764. 00000020:00000001:3.0:1784948732.340960:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000040:1.0:1784948732.340966:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000600c5764 x1871651289648896 msgsize 224 00000020:00000001:3.0:1784948732.340968:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00100000:1.0:1784948732.340974:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:3.0:1784948732.340974:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948732.340981:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948732.340984:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784948732.340987:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:3.0:1784948732.340989:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948732.340995:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948732.340996:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948732.340998:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:1.0:1784948732.341001:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948732.341002:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948732.341007:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948732.341011:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948732.341014:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948732.341015:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00100000:0.0:1784948732.341019:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289648896 02000000:00000001:3.0:1784948732.341020:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:0.0:1784948732.341024:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948732.341027:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000010:3.0:1784948732.341028:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000005b888794. 00000100:00000001:0.0:1784948732.341031:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948732.341032:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948732.341036:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.341037:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948732.341043:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289648896 00000020:00000001:3.0:1784948732.341044:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948732.341049:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00010000:00000040:3.0:1784948732.341050:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289648768 00000020:00000040:0.0:1784948732.341052:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000001:3.0:1784948732.341054:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:0.0:1784948732.341055:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948732.341059:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000200:3.0:1784948732.341060:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000028574ea9 x1871651289648768/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:393/0 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948732.341063:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948732.341069:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948732.341074:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:3.0:1784948732.341079:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:0.0:1784948732.341080:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000025dbf3d7. 00010000:00000001:3.0:1784948732.341082:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948732.341084:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00001000:3.0:1784948732.341087:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=54 v=5 (1 1 1 1) 00000020:00000010:0.0:1784948732.341088:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000025dbf3d7. 00000020:00000010:0.0:1784948732.341096:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000069d2b91d. 00000100:00000001:3.0:1784948732.341097:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948732.341102:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000010:0.0:1784948732.341104:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ae8262ed. 00000100:00000001:3.0:1784948732.341104:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000010:0.0:1784948732.341111:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 02000000:00000001:3.0:1784948732.341111:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948732.341113:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.341118:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948732.341122:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:3.0:1784948732.341122:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948732.341126:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000100:00000001:0.0:1784948732.341127:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948732.341131:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:3.0:1784948732.341133:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289648768, offset 224 00000100:00000001:0.0:1784948732.341134:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948732.341141:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948732.341142:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948732.341146:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948732.341148:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948732.341150:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948732.341158:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6434e80 00000100:00000001:0.0:1784948732.341160:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948732.341165:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x80d5 [1] + 224 00000400:00000200:3.0:1784948732.341184:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948732.341187:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784948732.341189:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948732.341191:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948732.341195:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784948732.341200:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 667 00000100:00000200:3.0:1784948732.341203:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000098475f18 x1871651289648768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948748 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948732.341206:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948732.341213:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451659520 : -115544257892096 : ffff96e9c18d7100) 00000100:00000040:0.0:1784948732.341225:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000600c5764 x1871651289648896/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:393/0 lens 224/0 e 0 to 0 dl 1784948743 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948732.341233:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000098475f18 x1871651289648768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948748 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948732.341245:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948732.341252:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000025dbf3d7. 00000100:00000001:3.0:1784948732.341253:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948732.341259:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948732.341264:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000025dbf3d7. 00000400:00000200:3.0:1784948732.341264:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:3.0:1784948732.341268:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:3.0:1784948732.341274:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948732.341277:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00100000:0.0:1784948732.341281:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000600c5764 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289648896:12345-192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948732.341285:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948732.341293:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000200:0.0:1784948732.341294:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289648896 00010000:00000001:3.0:1784948732.341301:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948732.341302:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948732.341304:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948732.341307:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948732.341309:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:0.0:1784948732.341311:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948732.341315:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:3.0:1784948732.341317:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000028574ea9 x1871651289648768/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:393/0 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948732.341320:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948732.341327:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948732.341332:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948732.341338:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948732.341341:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948732.341345:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948732.341351:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:3.0:1784948732.341353:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000028574ea9 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289648768:12345-0@lo:400: Request processed in 478us (858us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948732.341356:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948732.341360:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948732.341366:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002bfeb046. 00000100:00100000:3.0:1784948732.341370:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 691 02000000:00000001:0.0:1784948732.341372:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948732.341377:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948732.341378:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.341384:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:0.0:1784948732.341386:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948732.341388:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00010000:00000040:0.0:1784948732.341389:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289648896 00010000:00000001:0.0:1784948732.341393:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000010:3.0:1784948732.341395:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013785842. 00010000:00000200:0.0:1784948732.341401:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000600c5764 x1871651289648896/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:393/0 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948732.341402:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000020:00000010:3.0:1784948732.341410:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00010000:00000001:0.0:1784948732.341420:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000040:3.0:1784948732.341420:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000001:0.0:1784948732.341426:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948732.341427:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948732.341431:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000005b888794. 00000100:00001000:0.0:1784948732.341433:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=49 v=5 (1 1 1 1) 02000000:00000001:3.0:1784948732.341439:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948732.341440:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784948732.341444:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948732.341447:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948732.341453:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000200:3.0:1784948732.341453:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:0.0:1784948732.341461:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:3.0:1784948732.341461:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fe908aa6. 02000000:00000001:0.0:1784948732.341466:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948732.341470:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d1142873. 00000100:00000001:0.0:1784948732.341471:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948732.341480:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948732.341486:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000100:00000200:0.0:1784948732.341493:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289648896, offset 224 00000400:00000200:0.0:1784948732.341505:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948732.341518:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948732.341528:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3149:3149:249:4294967295] 192.168.202.137@tcp LPNI seq info [2056:2056:8:4294967295] 00000400:00000200:0.0:1784948732.341540:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948732.341548:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948732.341554:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000eafb2cdc. 00000800:00000200:0.0:1784948732.341563:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948732.341573:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948732.341577:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000eafb2cdc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948732.341592:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948732.341600:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948732.341607:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948732.341609:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948732.341614:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948732.341619:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000600c5764 x1871651289648896/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:393/0 lens 224/224 e 0 to 0 dl 1784948743 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948732.341638:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000600c5764 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289648896:12345-192.168.202.137@tcp:400: Request processed in 389us (667us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948732.341652:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 667 00000100:00000040:0.0:1784948732.341657:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948732.341661:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948732.341665:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948732.341669:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ae8262ed. 00000020:00000010:0.0:1784948732.341673:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948732.341676:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000069d2b91d. 00000020:00000040:0.0:1784948732.341680:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948732.341683:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948732.341693:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948732.341697:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000eafb2cdc. 00000400:00000200:3.0:1784948732.341701:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948732.341706:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948732.341709:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:3.0:1784948732.341711:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000100:00000001:3.0:1784948732.341714:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948732.341715:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948732.341720:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948732.341724:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948732.341726:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948732.341729:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948732.341733:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948732.341739:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6434f00 00000400:00000200:3.0:1784948732.341744:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x80dd [1] + 224 00000800:00000001:3.0:1784948732.341747:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948732.341754:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948732.341755:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948732.341758:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948732.341760:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948732.341763:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b8226094 x1871651289648896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948748 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948732.341769:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b8226094 x1871651289648896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948748 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948732.341782:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948732.341785:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948732.341787:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948732.382985:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305671722, at 4305671723 00000100:00000040:1.0:1784948732.382985:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305671722, at 4305671723 00000100:00000040:3.0:1784948732.382993:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305671722, at 4305671723 00000100:00000010:1.0:1784948732.383002:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00000100:00000010:0.0:1784948732.383005:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000010de4e21. 00000100:00000010:3.0:1784948732.383015:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00000800:00000100:0.0:1784948732.532040:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.10@tcp 00000800:00000010:0.0:1784948732.532046:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b0d293c7. 00000400:00000200:0.0:1784948732.532056:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948732.532064:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948732.532071:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11005 health :900 00000400:00000200:0.0:1784948732.532076:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948732.532080:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:0.0:1784948732.532087:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000800:00000100:0.0:1784948732.532111:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000800:00000010:0.0:1784948732.532114:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008746b1e6. 00000400:00000200:0.0:1784948732.532117:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948732.532121:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948732.532125:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948732.532128:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948732.532129:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:0.0:1784948732.532135:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:3.0:1784948732.532144:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000001:3.0:1784948732.532154:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948732.532164:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948732.532168:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:3.0:1784948732.532189:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000400:00000200:3.0:1784948732.532195:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948732.532200:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:3.0:1784948732.532205:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000015713dee. 00000400:00000200:3.0:1784948732.532213:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-110 00000400:00000200:3.0:1784948732.532216:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -110 00000400:00000200:3.0:1784948732.532221:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00000400:00000200:3.0:1784948732.532225:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:3.0:1784948732.532229:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948732.532230:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948732.532232:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000400:00000010:3.0:1784948732.532234:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000400:00000200:3.0:1784948732.532236:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948732.532240:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:3.0:1784948732.532243:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000194fc0c0. 00000400:00000200:3.0:1784948732.532247:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:3.0:1784948732.532249:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:3.0:1784948732.532253:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00000020:00000001:3.0:1784948733.172132:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948733.172137:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948733.172140:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948733.363997:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948733.364010:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948733.364011:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948733.364015:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948733.364017:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948733.364019:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948733.364021:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948733.556049:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:0.0:1784948733.556055:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000800:00000010:0.0:1784948733.556061:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006487a147. 00000400:00000010:1.0:1784948733.556066:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000089a38f57. 00000400:00000200:0.0:1784948733.556071:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:1.0:1784948733.556078:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000047e19e91. 00000400:00000200:0.0:1784948733.556079:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948733.556090:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11006 health :700 00000400:00000010:1.0:1784948733.556090:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000200:0.0:1784948733.556097:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948733.556102:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000010:1.0:1784948733.556104:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006d2c4b24. 00000400:00000200:0.0:1784948733.556109:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:1.0:1784948733.556119:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948733.556139:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948733.556148:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948733.556155:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3150:3150:251:4294967295] 192.168.202.137@tcp LPNI seq info [2057:2057:8:4294967295] 00000400:00000200:1.0:1784948733.556220:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948733.556247:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948733.556255:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948733.556262:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000057c7ab4. 00000400:00000200:3.0:1784948733.556268:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:1.0:1784948733.556276:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948733.556283:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000800:00000200:1.0:1784948733.556292:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:3.0:1784948733.556294:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1784948733.556300:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000057c7ab4 type 2, nob 96 niov 1 nkiov 0 00000400:00000001:3.0:1784948733.556300:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948733.556304:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:3.0:1784948733.556310:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000400:00000200:3.0:1784948733.556316:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948733.556323:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:3.0:1784948733.556328:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003cdea007. 00000400:00000200:3.0:1784948733.556336:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:3.0:1784948733.556340:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:3.0:1784948733.556347:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00000800:00000001:1.0:1784948733.556500:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948733.556515:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948733.556520:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948733.556526:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948733.556534:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948733.556549:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948733.556560:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948733.556571:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948733.556577:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3151:3151:252:4294967295] 192.168.202.137@tcp LPNI seq info [2058:2058:8:4294967295] 00000400:00000200:1.0:1784948733.556589:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948733.556595:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948733.556600:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d104522b. 00000800:00000200:1.0:1784948733.556606:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948733.556612:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948733.556616:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d104522b type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784948733.556663:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948733.556743:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948733.556749:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d104522b. 00000400:00000200:1.0:1784948733.556758:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948733.556775:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948733.556804:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948734.196193:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948734.196199:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948734.196220:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948734.388038:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948734.388055:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948734.388057:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948734.388063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948734.388067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948734.388071:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948734.388075:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948734.580080:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.25@tcp 00000800:00000010:0.0:1784948734.580092:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002aeee8e9. 00000400:00000200:0.0:1784948734.580106:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948734.580116:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.25@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948734.580127:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11007 health :900 00000400:00000200:0.0:1784948734.580137:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948734.580143:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.25@tcp: 1 00000400:00000200:0.0:1784948734.580151:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.25@tcp: -114 00000800:00000100:0.0:1784948734.580223:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000800:00000010:0.0:1784948734.580227:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006069fb41. 00000400:00000200:0.0:1784948734.580236:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948734.580246:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948734.580254:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948734.580261:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:3.0:1784948734.580268:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948734.580271:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:3.0:1784948734.580290:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.25@tcp(000000003bd24629) state 0x840 00000400:00000001:3.0:1784948734.580304:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.580309:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.580314:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:3.0:1784948734.580320:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000400:00000200:3.0:1784948734.580330:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.580338:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.25@tcp: -114 00000400:00000010:3.0:1784948734.580345:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089e37819. 00000400:00000200:3.0:1784948734.580353:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.25@tcp:-110 00000400:00000200:3.0:1784948734.580357:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.25@tcp(000000003bd24629) state 0x2040 rc -110 00000400:00000200:3.0:1784948734.580361:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.25@tcp 00000400:00000200:3.0:1784948734.580380:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:3.0:1784948734.580384:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.580386:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.580387:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000400:00000010:3.0:1784948734.580390:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000400:00000200:3.0:1784948734.580392:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.580395:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:3.0:1784948734.580399:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000070ada26c. 00000400:00000200:3.0:1784948734.580403:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000200:3.0:1784948734.580406:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:3.0:1784948734.580409:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:1.0:1784948734.580494:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.25@tcp NID 111.222.173.25@tcp: -110. discovery complete 00000400:00000200:1.0:1784948734.580503:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.173.25@tcp primary NID 111.222.173.25@tcp rc -110 00000100:00000010:1.0:1784948734.580524:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 0000000030655a79. 00000100:00000001:1.0:1784948734.580556:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:1.0:1784948734.580560:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=0000000030655a79 refcount 1 to 111.222.173.25@tcp 00010000:00000010:1.0:1784948734.580566:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000052b5f963. 00010000:00080000:1.0:1784948734.580618:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.173.25@tcp at tail 00010000:00000001:1.0:1784948734.580624:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948734.580631:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948734.580648:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 000000007ed348a7. 00000100:00000400:1.0:1784948734.580657:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.142.25@tcp: rc = -75 00000100:00000040:1.0:1784948734.580663:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.142.25@tcp cnt=32 00000400:00000200:1.0:1784948734.580668:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:1.0:1784948734.580672:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:1.0:1784948734.580678:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:1.0:1784948734.580681:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:1.0:1784948734.580685:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:1.0:1784948734.580696:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1784948734.580705:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.580710:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.580715:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:1.0:1784948734.580719:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:1.0:1784948734.580725:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.580756:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:1.0:1784948734.580761:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.580764:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.580769:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.580774:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:1.0:1784948734.580779:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:1.0:1784948734.580786:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.580800:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:1.0:1784948734.580803:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948734.580813:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784948734.580814:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.580827:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.580834:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784948734.580841:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:1.0:1784948734.580843:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000010:3.0:1784948734.580856:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005e186afc. 00000400:00000200:1.0:1784948734.580858:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000010:3.0:1784948734.580868:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000400:00000200:1.0:1784948734.580876:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:1.0:1784948734.580883:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:3.0:1784948734.580890:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007701057d. 00000400:00000200:1.0:1784948734.580891:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.580933:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:1.0:1784948734.580967:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.580974:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:1.0:1784948734.580981:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:3.0:1784948734.580989:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:1.0:1784948734.580994:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581002:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.581010:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.581015:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.581018:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:3.0:1784948734.581023:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3152:3152:254:4294967295] 111.222.173.1@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:3.0:1784948734.581033:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:3.0:1784948734.581042:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.581049:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000800:00000010:3.0:1784948734.581056:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e3ed36ac. 00000400:00000200:1.0:1784948734.581060:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:1.0:1784948734.581068:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:3.0:1784948734.581071:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:1.0:1784948734.581078:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:3.0:1784948734.581079:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948734.581095:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:1.0:1784948734.581099:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000800:00000200:3.1:1784948734.581102:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000400:00000200:1.0:1784948734.581111:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000800:00000100:3.0:1784948734.581119:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.1@tcp 00000800:00000010:3.0:1784948734.581128:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e3ed36ac. 00000400:00000200:1.0:1784948734.581129:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784948734.581135:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948734.581142:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581151:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948734.581186:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:1.0:1784948734.581190:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581196:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.581198:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948734.581202:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784948734.581210:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:1.0:1784948734.581211:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:1.0:1784948734.581221:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000200:3.0:1784948734.581226:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:1.0:1784948734.581233:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581246:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:1.0:1784948734.581251:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000400:00000200:3.0:1784948734.581258:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 rc 0 00000400:00000200:1.0:1784948734.581260:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581269:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.581276:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581279:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:3.0:1784948734.581286:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:1.0:1784948734.581287:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:1.0:1784948734.581300:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000010:3.0:1784948734.581303:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000200:1.0:1784948734.581310:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000010:3.0:1784948734.581313:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000400:00000200:1.0:1784948734.581315:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581323:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.581326:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581329:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000010:3.0:1784948734.581331:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000cc444ae0. 00000400:00000200:1.0:1784948734.581333:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:1.0:1784948734.581342:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581351:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:1.0:1784948734.581357:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:1.0:1784948734.581360:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581365:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.581368:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581370:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:1.0:1784948734.581374:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000200:3.0:1784948734.581380:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:1.0:1784948734.581387:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581393:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.581400:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.581403:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.581405:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:3.0:1784948734.581408:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3153:3153:254:4294967295] 111.222.142.1@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:3.0:1784948734.581414:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:3.0:1784948734.581420:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.581426:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000800:00000010:3.0:1784948734.581431:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e3ed36ac. 00000400:00000200:1.0:1784948734.581434:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:1.0:1784948734.581440:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581446:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:3.0:1784948734.581446:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:1.0:1784948734.581457:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000800:00000200:3.1:1784948734.581466:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:1.0:1784948734.581468:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:1.0:1784948734.581477:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:3.0:1784948734.581482:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:1.0:1784948734.581490:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581495:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:1.0:1784948734.581503:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:1.0:1784948734.581508:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581513:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.581521:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:1.0:1784948734.581522:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581528:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000200:1.0:1784948734.581535:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000200:1.0:1784948734.581539:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000010:3.0:1784948734.581541:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000013f68999. 00000400:00000200:1.0:1784948734.581546:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:1.0:1784948734.581553:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:3.0:1784948734.581555:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000400:00000200:1.0:1784948734.581556:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.581566:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581570:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:1.0:1784948734.581572:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000010:3.0:1784948734.581573:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002b26853b. 00000400:00000200:1.0:1784948734.581578:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.581594:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:1.0:1784948734.581598:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948734.581599:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:1.0:1784948734.581601:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.581610:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581615:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:1.0:1784948734.581617:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:1.0:1784948734.581627:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581629:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:3.0:1784948734.581639:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.581645:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.581649:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.581654:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:3.0:1784948734.581659:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3154:3154:253:4294967295] 111.222.173.2@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:3.0:1784948734.581668:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:3.0:1784948734.581678:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.581689:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000400:00000200:1.0:1784948734.581694:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:1.0:1784948734.581699:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581701:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000010:3.0:1784948734.581703:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d9530b6c. 00000400:00000200:1.0:1784948734.581706:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581715:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:1.0:1784948734.581723:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000800:00000200:3.0:1784948734.581727:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:1.0:1784948734.581731:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.581743:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:1.0:1784948734.581747:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581752:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:3.1:1784948734.581755:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:1.0:1784948734.581762:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581767:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:1.0:1784948734.581774:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:3.0:1784948734.581774:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:1.0:1784948734.581785:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581791:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:1.0:1784948734.581795:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:1.0:1784948734.581803:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581810:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.581817:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:1.0:1784948734.581818:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581832:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:1.0:1784948734.581834:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:1.0:1784948734.581837:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000010:3.0:1784948734.581841:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eb3e5fa2. 00000400:00000200:1.0:1784948734.581844:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000400:00000200:1.0:1784948734.581851:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581854:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:3.0:1784948734.581856:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000400:00000200:1.0:1784948734.581859:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581868:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:1.0:1784948734.581871:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000010:3.0:1784948734.581874:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000399be696. 00000400:00000200:1.0:1784948734.581876:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.581896:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784948734.581900:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:1.0:1784948734.581901:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581933:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.581938:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581941:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:1.0:1784948734.581943:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:1.0:1784948734.581946:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.581959:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:1.0:1784948734.581962:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.581966:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.581969:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.581972:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:1.0:1784948734.581976:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:3.0:1784948734.581982:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:1.0:1784948734.581985:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.581995:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.582003:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.582008:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.582013:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:3.0:1784948734.582018:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3155:3155:252:4294967295] 111.222.142.2@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:3.0:1784948734.582028:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:3.0:1784948734.582038:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.582048:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000400:00000200:1.0:1784948734.582053:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:1.0:1784948734.582059:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:3.0:1784948734.582059:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f579e76d. 00000400:00000200:1.0:1784948734.582065:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582075:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582082:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000800:00000200:3.0:1784948734.582084:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:1.0:1784948734.582088:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:1.0:1784948734.582103:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000800:00000200:3.1:1784948734.582108:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:1.0:1784948734.582115:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:1.0:1784948734.582121:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.582124:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582127:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582130:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:1.0:1784948734.582132:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:1.0:1784948734.582136:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.582140:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:1.0:1784948734.582142:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.582143:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582146:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582149:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:1.0:1784948734.582151:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:1.0:1784948734.582172:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.582179:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784948734.582182:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:1.0:1784948734.582183:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.582193:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.582199:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x261 rc 0 00000400:00000200:1.0:1784948734.582208:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582215:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:1.0:1784948734.582218:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:3.0:1784948734.582224:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:1.0:1784948734.582229:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.582239:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000010:3.0:1784948734.582241:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004041a48e. 00000400:00000200:1.0:1784948734.582242:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.582247:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582255:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000010:3.0:1784948734.582259:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000400:00000200:1.0:1784948734.582260:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:1.0:1784948734.582269:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:1.0:1784948734.582278:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000010:3.0:1784948734.582279:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000905e873. 00000400:00000200:1.0:1784948734.582286:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:1.0:1784948734.582290:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.582297:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.582298:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:1.0:1784948734.582307:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582311:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:1.0:1784948734.582314:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:1.0:1784948734.582324:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.582327:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:3.0:1784948734.582337:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.582343:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.582348:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.582354:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:3.0:1784948734.582360:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3156:3156:251:4294967295] 111.222.173.3@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:3.0:1784948734.582370:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:3.0:1784948734.582380:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.582389:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000400:00000200:1.0:1784948734.582395:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000800:00000010:3.0:1784948734.582398:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004f41fdbc. 00000400:00000200:1.0:1784948734.582400:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.582407:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582421:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000800:00000200:3.0:1784948734.582425:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:1.0:1784948734.582427:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000200:1.0:1784948734.582436:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000800:00000200:3.0:1784948734.582441:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000400:00000200:1.0:1784948734.582445:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948734.582458:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:1.0:1784948734.582462:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:3.1:1784948734.582469:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:1.0:1784948734.582470:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582480:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000800:00000200:3.1:1784948734.582482:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000400:00000200:1.0:1784948734.582485:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:1.0:1784948734.582493:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:1.0:1784948734.582506:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000800:00000100:3.0:1784948734.582512:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.3@tcp 00000400:00000200:1.0:1784948734.582516:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:1.0:1784948734.582522:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:3.0:1784948734.582522:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004f41fdbc. 00000400:00000200:1.0:1784948734.582528:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.582539:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948734.582552:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948734.582621:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582627:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000200:3.0:1784948734.582631:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948734.582632:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:1.0:1784948734.582644:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.582651:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:1.0:1784948734.582659:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:1.0:1784948734.582664:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948734.582675:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:1.0:1784948734.582684:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582691:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582695:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:1.0:1784948734.582699:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:1.0:1784948734.582704:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948734.582710:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:1.0:1784948734.582717:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:1.0:1784948734.582723:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948734.582725:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 rc 0 00000400:00000200:1.0:1784948734.582728:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948734.582738:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:1.0:1784948734.582743:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:1.0:1784948734.582746:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000200:3.0:1784948734.582752:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000200:1.0:1784948734.582756:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000001:3.0:1784948734.582769:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1784948734.582771:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000001:3.0:1784948734.582777:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948734.582778:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948734.582784:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948734.582787:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000200:1.0:1784948734.582794:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000010:3.0:1784948734.582797:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000400:00000200:1.0:1784948734.582800:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000100:00000010:1.0:1784948734.582808:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 000000007ed348a7. 00000400:00000200:3.0:1784948734.582811:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000020:00000001:1.0:1784948734.582821:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000200:3.0:1784948734.582826:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00010000:00000001:1.0:1784948734.582830:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000100:00000001:1.0:1784948734.582839:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000010:3.0:1784948734.582839:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005e186afc. 00000400:00000200:3.0:1784948734.582856:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-5 00000400:00000010:1.0:1784948734.582861:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 000000003b287909. 00000400:00000200:3.0:1784948734.582864:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -5 00000400:00000200:1.0:1784948734.582873:0:343529:0:(peer.c:266:lnet_peer_alloc()) 000000003b287909 nid 111.222.142.25@tcp 00000400:00000010:1.0:1784948734.582880:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000025674e46. 00000400:00000200:3.0:1784948734.582882:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000400:00000200:1.0:1784948734.582889:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 0000000025674e46 net tcp 00000400:00000010:1.0:1784948734.582895:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000007465fe15. 00000400:00000200:3.0:1784948734.582901:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:1.0:1784948734.582932:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 000000007465fe15 nid 111.222.142.25@tcp 00000400:00000200:1.0:1784948734.582943:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.142.25@tcp NID 111.222.142.25@tcp flags 0x0 00000400:00000200:1.0:1784948734.582947:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.142.25@tcp: 0 00000400:00000200:1.0:1784948734.582955:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.25@tcp: 0 00000400:00000200:1.0:1784948734.582962:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:3.0:1784948734.582972:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005e186afc. 00000400:00000010:3.0:1784948734.582982:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000400:00000010:3.0:1784948734.582993:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ae8b415c. 00000400:00000200:3.0:1784948734.583004:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784948734.583018:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:3.0:1784948734.583027:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.583033:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.583040:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.583046:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:3.0:1784948734.583053:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3157:3157:251:4294967295] 111.222.142.3@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:3.0:1784948734.583062:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:3.0:1784948734.583072:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.583080:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:3.0:1784948734.583085:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004f41fdbc. 00000800:00000200:3.0:1784948734.583093:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:3.1:1784948734.583101:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784948734.583107:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:3.0:1784948734.583111:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x261 rc 0 00000400:00000200:3.0:1784948734.583118:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000010:3.0:1784948734.583124:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000cf77ef26. 00000400:00000010:3.0:1784948734.583128:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000400:00000010:3.0:1784948734.583134:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b666dac0. 00000400:00000200:3.0:1784948734.583140:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784948734.583147:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:3.0:1784948734.583151:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.583185:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.583190:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.583194:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:3.0:1784948734.583199:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3158:3158:250:4294967295] 111.222.173.4@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:3.0:1784948734.583206:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:3.0:1784948734.583216:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.583223:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000800:00000010:3.0:1784948734.583229:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f538dcf6. 00000800:00000200:3.0:1784948734.583237:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:3.1:1784948734.583243:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784948734.583250:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:3.0:1784948734.583253:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784948734.583259:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000010:3.0:1784948734.583266:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ee879bbd. 00000400:00000010:3.0:1784948734.583270:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000400:00000010:3.0:1784948734.583276:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000017f0d388. 00000400:00000200:3.0:1784948734.583282:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:3.0:1784948734.583289:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:3.0:1784948734.583294:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.583298:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.583302:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.583306:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:3.0:1784948734.583311:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3159:3159:249:4294967295] 111.222.142.4@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:3.0:1784948734.583318:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:3.0:1784948734.583327:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.583335:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:3.0:1784948734.583341:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ea573232. 00000800:00000200:3.0:1784948734.583348:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:3.1:1784948734.583354:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:3.0:1784948734.583380:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:3.0:1784948734.583384:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x261 rc 0 00000400:00000200:3.0:1784948734.583389:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000010:3.0:1784948734.583396:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004f865a97. 00000400:00000010:3.0:1784948734.583401:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000400:00000010:3.0:1784948734.583407:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000039ee8127. 00000400:00000200:3.0:1784948734.583413:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:3.0:1784948734.583423:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:3.0:1784948734.583430:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.583434:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.583439:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.583445:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:3.0:1784948734.583451:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3160:3160:248:4294967295] 111.222.173.5@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:3.0:1784948734.583459:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:3.0:1784948734.583473:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.583482:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000800:00000010:3.0:1784948734.583489:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000075354f3d. 00000800:00000200:3.0:1784948734.583498:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.0:1784948734.583502:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948734.583512:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:3.1:1784948734.583516:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1784948734.583524:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.5@tcp 00000800:00000010:3.0:1784948734.583527:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000075354f3d. 00000400:00000200:3.0:1784948734.583532:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948734.583538:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948734.583545:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948734.583550:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:3.0:1784948734.583556:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:3.0:1784948734.583567:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:3.0:1784948734.583574:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 rc 0 00000400:00000200:3.0:1784948734.583581:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:3.0:1784948734.583588:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006841dfc4. 00000400:00000010:3.0:1784948734.583593:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000400:00000010:3.0:1784948734.583599:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a400fa42. 00000400:00000200:3.0:1784948734.583605:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:3.0:1784948734.583612:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:3.0:1784948734.583617:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.583621:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.583625:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.583629:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:3.0:1784948734.583635:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3161:3161:248:4294967295] 111.222.173.6@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:3.0:1784948734.583642:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:3.0:1784948734.583651:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.583659:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:3.0:1784948734.583664:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000075354f3d. 00000800:00000200:3.0:1784948734.583671:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:3.1:1784948734.583679:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:3.0:1784948734.583685:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:3.0:1784948734.583688:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:3.0:1784948734.583693:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:3.0:1784948734.583701:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f0e4e38. 00000400:00000010:3.0:1784948734.583706:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 00000400:00000010:3.0:1784948734.583712:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009b1df9a1. 00000400:00000200:3.0:1784948734.583718:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:3.0:1784948734.583726:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:3.0:1784948734.583731:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.583735:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.583739:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.583743:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:3.0:1784948734.583748:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3162:3162:247:4294967295] 111.222.142.6@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:3.0:1784948734.583755:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:3.0:1784948734.583766:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.583775:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:3.0:1784948734.583781:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000efb7ac25. 00000800:00000200:3.0:1784948734.583788:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:3.1:1784948734.583795:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000200:3.0:1784948734.583822:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:3.0:1784948734.583826:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x261 rc 0 00000400:00000200:3.0:1784948734.583834:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:3.0:1784948734.583845:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000041892d66. 00000400:00000010:3.0:1784948734.583851:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000400:00000010:3.0:1784948734.583863:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f99d1b9f. 00000400:00000200:3.0:1784948734.583871:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:3.0:1784948734.583883:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:3.0:1784948734.583891:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.583896:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.583901:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.583965:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:3.0:1784948734.583973:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3163:3163:246:4294967295] 111.222.173.7@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:3.0:1784948734.583983:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:3.0:1784948734.583999:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.584009:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:3.0:1784948734.584017:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a219b995. 00000800:00000200:3.0:1784948734.584026:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.0:1784948734.584031:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948734.584041:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:3.1:1784948734.584046:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1784948734.584055:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.7@tcp 00000800:00000010:3.0:1784948734.584059:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a219b995. 00000400:00000200:3.0:1784948734.584065:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948734.584071:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948734.584077:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948734.584081:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:3.0:1784948734.584086:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:3.0:1784948734.584098:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:3.0:1784948734.584101:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 rc 0 00000400:00000200:3.0:1784948734.584108:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:3.0:1784948734.584116:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e5260462. 00000400:00000010:3.0:1784948734.584121:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bb55de7f. 00000400:00000010:3.0:1784948734.584128:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000534ace9d. 00000400:00000200:3.0:1784948734.584134:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:3.0:1784948734.584145:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:3.0:1784948734.584150:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.584185:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.584190:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.584195:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:3.0:1784948734.584199:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3164:3164:246:4294967295] 111.222.142.7@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:3.0:1784948734.584208:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:3.0:1784948734.584218:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.584225:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:3.0:1784948734.584232:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a219b995. 00000800:00000200:3.0:1784948734.584240:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:3.1:1784948734.584247:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:3.0:1784948734.584253:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:3.0:1784948734.584257:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:3.0:1784948734.584263:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:3.0:1784948734.584269:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007a6587e5. 00000400:00000010:3.0:1784948734.584275:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000400:00000010:3.0:1784948734.584281:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000074570d70. 00000400:00000200:3.0:1784948734.584288:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:3.0:1784948734.584296:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:3.0:1784948734.584301:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.584307:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.584311:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.584315:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:3.0:1784948734.584320:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3165:3165:245:4294967295] 111.222.173.8@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:3.0:1784948734.584328:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:3.0:1784948734.584337:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.584345:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:3.0:1784948734.584352:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000071cce8cf. 00000800:00000200:3.0:1784948734.584359:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:3.1:1784948734.584367:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:3.0:1784948734.584372:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:3.0:1784948734.584376:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:3.0:1784948734.584382:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:3.0:1784948734.584388:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003f3c01c8. 00000400:00000010:3.0:1784948734.584392:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000400:00000010:3.0:1784948734.584398:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007174c252. 00000400:00000200:3.0:1784948734.584403:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:3.0:1784948734.584410:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:3.0:1784948734.584415:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.584419:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.584424:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.584428:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:3.0:1784948734.584433:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3166:3166:244:4294967295] 111.222.142.8@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:3.0:1784948734.584440:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:3.0:1784948734.584449:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.584456:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:3.0:1784948734.584460:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000089a028d. 00000800:00000200:3.0:1784948734.584467:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:3.1:1784948734.584476:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000400:00000200:3.0:1784948734.584502:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:3.0:1784948734.584506:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x261 rc 0 00000400:00000200:3.0:1784948734.584512:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:3.0:1784948734.584519:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000063f28b6f. 00000400:00000010:3.0:1784948734.584523:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000400:00000010:3.0:1784948734.584530:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c0a3f621. 00000400:00000200:3.0:1784948734.584536:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:3.0:1784948734.584546:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:3.0:1784948734.584554:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.584560:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.584565:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.584570:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:3.0:1784948734.584577:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3167:3167:243:4294967295] 111.222.173.9@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:3.0:1784948734.584587:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:3.0:1784948734.584600:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.584609:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:3.0:1784948734.584617:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004805f207. 00000800:00000200:3.0:1784948734.584624:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.0:1784948734.584629:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948734.584639:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:3.1:1784948734.584642:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1784948734.584649:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.9@tcp 00000800:00000010:3.0:1784948734.584653:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004805f207. 00000400:00000200:3.0:1784948734.584660:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948734.584668:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948734.584674:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948734.584679:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:3.0:1784948734.584688:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:3.0:1784948734.584699:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:3.0:1784948734.584703:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 rc 0 00000400:00000200:3.0:1784948734.584709:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:3.0:1784948734.584716:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000072ce2c8. 00000400:00000010:3.0:1784948734.584721:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000400:00000010:3.0:1784948734.584728:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b7bd3e11. 00000400:00000200:3.0:1784948734.584733:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:3.0:1784948734.584742:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:3.0:1784948734.584747:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.584752:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.584757:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.584761:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:3.0:1784948734.584767:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3168:3168:243:4294967295] 111.222.142.9@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:3.0:1784948734.584775:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:3.0:1784948734.584785:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.584794:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:3.0:1784948734.584800:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004805f207. 00000800:00000200:3.0:1784948734.584807:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:3.1:1784948734.584814:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:3.0:1784948734.584820:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:3.0:1784948734.584824:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:3.0:1784948734.584831:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:3.0:1784948734.584838:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ec8543b3. 00000400:00000010:3.0:1784948734.584844:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000467f058f. 00000400:00000010:3.0:1784948734.584850:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000079f1d10e. 00000400:00000200:3.0:1784948734.584856:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:3.0:1784948734.584863:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:3.0:1784948734.584869:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.584873:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.584878:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.584882:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:3.0:1784948734.584888:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3169:3169:242:4294967295] 111.222.173.10@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:3.0:1784948734.584896:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:3.0:1784948734.584966:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.584978:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:3.0:1784948734.584985:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000040401747. 00000800:00000200:3.0:1784948734.584994:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:3.1:1784948734.585002:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:3.0:1784948734.585011:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:3.0:1784948734.585017:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:3.0:1784948734.585024:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:3.0:1784948734.585035:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001879354a. 00000400:00000010:3.0:1784948734.585041:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000400:00000010:3.0:1784948734.585048:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000003ac0766. 00000400:00000200:3.0:1784948734.585053:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:3.0:1784948734.585062:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:3.0:1784948734.585067:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.585072:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.585076:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.585080:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:3.0:1784948734.585086:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3170:3170:241:4294967295] 111.222.142.10@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:3.0:1784948734.585094:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:3.0:1784948734.585105:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.585112:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:3.0:1784948734.585118:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003264a2f8. 00000800:00000200:3.0:1784948734.585125:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:3.1:1784948734.585131:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000400:00000200:3.0:1784948734.585183:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:3.0:1784948734.585188:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x261 rc 0 00000400:00000200:3.0:1784948734.585198:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:3.0:1784948734.585207:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dc7a25c6. 00000400:00000010:3.0:1784948734.585213:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000400:00000010:3.0:1784948734.585223:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000077b9b581. 00000400:00000200:3.0:1784948734.585232:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:3.0:1784948734.585244:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:3.0:1784948734.585251:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.585257:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.585262:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.585267:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:3.0:1784948734.585272:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3171:3171:240:4294967295] 111.222.173.11@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:3.0:1784948734.585281:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:3.0:1784948734.585292:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.585301:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:3.0:1784948734.585309:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000042e55fb6. 00000800:00000200:3.0:1784948734.585317:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.0:1784948734.585322:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948734.585332:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:3.1:1784948734.585337:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1784948734.585367:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.11@tcp 00000800:00000010:3.0:1784948734.585371:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000042e55fb6. 00000400:00000200:3.0:1784948734.585380:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948734.585387:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948734.585394:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948734.585399:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:3.0:1784948734.585405:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:3.0:1784948734.585417:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:3.0:1784948734.585421:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 rc 0 00000400:00000200:3.0:1784948734.585427:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:3.0:1784948734.585436:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000684a4a3c. 00000400:00000010:3.0:1784948734.585442:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000400:00000010:3.0:1784948734.585448:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000249338ff. 00000400:00000200:3.0:1784948734.585455:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:3.0:1784948734.585465:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:3.0:1784948734.585470:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.585474:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.585478:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.585484:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:3.0:1784948734.585490:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3172:3172:240:4294967295] 111.222.142.11@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:3.0:1784948734.585499:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:3.0:1784948734.585510:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.585519:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:3.0:1784948734.585524:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000042e55fb6. 00000800:00000200:3.0:1784948734.585533:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:3.1:1784948734.585540:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:3.0:1784948734.585546:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:3.0:1784948734.585550:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:3.0:1784948734.585556:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:3.0:1784948734.585563:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000eae26e4. 00000400:00000010:3.0:1784948734.585568:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000400:00000010:3.0:1784948734.585575:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000011546f5a. 00000400:00000200:3.0:1784948734.585581:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:3.0:1784948734.585588:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:3.0:1784948734.585593:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.585598:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.585602:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.585606:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:3.0:1784948734.585612:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3173:3173:239:4294967295] 111.222.173.12@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:3.0:1784948734.585621:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:3.0:1784948734.585630:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.585638:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:3.0:1784948734.585644:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b4c31151. 00000800:00000200:3.0:1784948734.585651:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:3.1:1784948734.585659:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:3.0:1784948734.585665:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:3.0:1784948734.585669:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:3.0:1784948734.585675:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:3.0:1784948734.585682:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000307ddfde. 00000400:00000010:3.0:1784948734.585687:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000400:00000010:3.0:1784948734.585694:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001e8db685. 00000400:00000200:3.0:1784948734.585700:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:3.0:1784948734.585709:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:3.0:1784948734.585713:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.585718:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.585722:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.585727:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:3.0:1784948734.585732:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3174:3174:238:4294967295] 111.222.142.12@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:3.0:1784948734.585740:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:3.0:1784948734.585749:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.585759:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:3.0:1784948734.585764:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a12a46b5. 00000800:00000200:3.0:1784948734.585771:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:3.1:1784948734.585779:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:3.0:1784948734.585805:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:3.0:1784948734.585809:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:3.0:1784948734.585816:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:3.0:1784948734.585843:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000010:3.0:1784948734.585852:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 00000400:00000010:3.0:1784948734.585863:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000cdcf0a74. 00000400:00000200:3.0:1784948734.585884:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:3.0:1784948734.585893:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:3.0:1784948734.585899:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.585905:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.585953:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.585961:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:3.0:1784948734.585970:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3175:3175:237:4294967295] 111.222.173.13@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:3.0:1784948734.585982:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:3.0:1784948734.585994:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.586002:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:3.0:1784948734.586009:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006c136176. 00000800:00000200:3.0:1784948734.586017:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.0:1784948734.586022:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.13 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948734.586030:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:3.1:1784948734.586034:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.13 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1784948734.586041:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.13@tcp 00000800:00000010:3.0:1784948734.586045:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006c136176. 00000400:00000200:3.0:1784948734.586052:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948734.586058:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948734.586064:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948734.586069:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:3.0:1784948734.586074:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:3.0:1784948734.586084:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:3.0:1784948734.586088:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 rc 0 00000400:00000200:3.0:1784948734.586095:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:3.0:1784948734.586103:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000059e5b358. 00000400:00000010:3.0:1784948734.586108:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000400:00000010:3.0:1784948734.586114:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000cd6cefc7. 00000400:00000200:3.0:1784948734.586120:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:3.0:1784948734.586128:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:3.0:1784948734.586134:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.586138:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.586144:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.586150:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:3.0:1784948734.586184:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3176:3176:237:4294967295] 111.222.142.13@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:3.0:1784948734.586192:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:3.0:1784948734.586203:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.586210:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:3.0:1784948734.586216:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006c136176. 00000800:00000200:3.0:1784948734.586224:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:3.1:1784948734.586231:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:3.0:1784948734.586237:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:3.0:1784948734.586240:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:3.0:1784948734.586247:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:3.0:1784948734.586258:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000061851253. 00000400:00000010:3.0:1784948734.586263:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000400:00000010:3.0:1784948734.586271:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d8357cb9. 00000400:00000200:3.0:1784948734.586277:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:3.0:1784948734.586285:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:3.0:1784948734.586289:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.586294:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.586298:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.586303:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:3.0:1784948734.586308:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3177:3177:236:4294967295] 111.222.173.14@tcp LPNI seq info [21:21:8:4294967295] 00000400:00000200:3.0:1784948734.586315:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:3.0:1784948734.586325:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.586333:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:3.0:1784948734.586339:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e5f52141. 00000800:00000200:3.0:1784948734.586346:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:3.1:1784948734.586353:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:3.0:1784948734.586360:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:3.0:1784948734.586363:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:3.0:1784948734.586370:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:3.0:1784948734.586377:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004b465f54. 00000400:00000010:3.0:1784948734.586382:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000400:00000010:3.0:1784948734.586390:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000dda44ac8. 00000400:00000200:3.0:1784948734.586396:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:3.0:1784948734.586405:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:3.0:1784948734.586410:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.586415:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.586419:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.586423:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:3.0:1784948734.586428:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3178:3178:235:4294967295] 111.222.142.14@tcp LPNI seq info [21:21:8:4294967295] 00000400:00000200:3.0:1784948734.586437:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:3.0:1784948734.586448:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.586455:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:3.0:1784948734.586460:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e6aac0b. 00000800:00000200:3.0:1784948734.586468:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:3.1:1784948734.586475:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:3.0:1784948734.586499:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:3.0:1784948734.586504:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:3.0:1784948734.586510:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:3.0:1784948734.586515:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.586518:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.586521:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:3.0:1784948734.586525:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000400:00000200:3.0:1784948734.586530:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.586535:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:3.0:1784948734.586540:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004041a48e. 00000400:00000200:3.0:1784948734.586548:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-5 00000400:00000200:3.0:1784948734.586552:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -5 00000400:00000200:3.0:1784948734.586557:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:3.0:1784948734.586563:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:3.0:1784948734.586570:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000208850ab. 00000400:00000010:3.0:1784948734.586575:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000400:00000010:3.0:1784948734.586585:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009b22b71e. 00000400:00000200:3.0:1784948734.586592:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:3.0:1784948734.586601:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:3.0:1784948734.586608:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.586615:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.586620:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.586626:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:3.0:1784948734.586632:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3179:3179:234:4294967295] 111.222.173.15@tcp LPNI seq info [20:20:8:4294967295] 00000400:00000200:3.0:1784948734.586640:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:3.0:1784948734.586651:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.586660:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:3.0:1784948734.586668:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948734.586676:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.0:1784948734.586681:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.15 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1784948734.586690:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:3.1:1784948734.586694:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.15 (cnted 0, interval 60, 11 secs later) 00000800:00000100:3.0:1784948734.586703:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.15@tcp 00000800:00000010:3.0:1784948734.586706:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:3.0:1784948734.586712:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948734.586719:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948734.586724:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948734.586729:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:3.0:1784948734.586734:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:3.0:1784948734.586744:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:3.0:1784948734.586748:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 rc 0 00000400:00000200:3.0:1784948734.586754:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:3.0:1784948734.586760:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004041a48e. 00000400:00000010:3.0:1784948734.586765:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000400:00000010:3.0:1784948734.586771:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002d12d4bf. 00000400:00000200:3.0:1784948734.586776:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:3.0:1784948734.586784:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:3.0:1784948734.586788:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948734.586792:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.586796:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948734.586799:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:3.0:1784948734.586805:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3180:3180:234:4294967295] 111.222.142.15@tcp LPNI seq info [19:19:8:4294967295] 00000400:00000200:3.0:1784948734.586813:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:3.0:1784948734.586823:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.586830:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:3.0:1784948734.586834:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948734.586841:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:3.1:1784948734.586849:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:3.0:1784948734.586855:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:3.0:1784948734.586860:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:3.0:1784948734.586866:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.25@tcp(000000003b287909) state 0x40 00000400:00000010:3.0:1784948734.586874:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000011003cc7. 00000400:00000010:3.0:1784948734.586879:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 00000400:00000010:3.0:1784948734.586885:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000061dbe3a3. 00000400:00000200:3.0:1784948734.586892:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.25@tcp 00000400:00000200:3.0:1784948734.586900:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.142.25@tcp local destination 00000400:00000200:3.0:1784948734.586947:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948734.586956:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.142.25@tcp 00000400:00000200:3.0:1784948734.586962:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.142.25@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:3.0:1784948734.586969:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3181:3181:233:4294967295] 111.222.142.25@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:3.0:1784948734.586978:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.25@tcp 00000400:00000200:3.0:1784948734.586988:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.25@tcp(111.222.142.25@tcp:111.222.142.25@tcp) : GET try# 0 00000800:00000200:3.0:1784948734.586996:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.25@tcp 00000800:00000010:3.0:1784948734.587001:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003547cd5f. 00000800:00000010:3.0:1784948734.587009:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000bc80ef80. 00000800:00000010:3.0:1784948734.587014:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 0000000055cd4eef. 00000800:00000200:3.0:1784948734.587019:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bc80ef80] -> 12345-111.222.142.25@tcp (2) 00000800:00000200:3.1:1784948734.587023:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bc80ef80] -> 12345-111.222.142.25@tcp (2) 00000400:00000200:3.0:1784948734.587042:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.25@tcp 00000400:00000200:3.0:1784948734.587045:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.25@tcp(000000003b287909) state 0x240 rc 0 00000400:00000200:3.0:1784948734.587049:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 00000400:00000001:3.0:1784948734.587053:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.587054:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.587057:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:3.0:1784948734.587059:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000400:00000200:3.0:1784948734.587062:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.587067:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:3.0:1784948734.587070:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004f865a97. 00000400:00000200:3.0:1784948734.587075:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-5 00000400:00000200:3.0:1784948734.587078:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -5 00000400:00000200:3.0:1784948734.587083:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:3.0:1784948734.587088:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000001:3.0:1784948734.587092:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.587094:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.587096:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:3.0:1784948734.587097:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000400:00000200:3.0:1784948734.587101:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.587105:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:3.0:1784948734.587107:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000041892d66. 00000400:00000200:3.0:1784948734.587114:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-5 00000400:00000200:3.0:1784948734.587118:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -5 00000400:00000200:3.0:1784948734.587121:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000400:00000200:3.0:1784948734.587124:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000001:3.0:1784948734.587126:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.587127:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.587129:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000400:00000010:3.0:1784948734.587131:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 00000400:00000200:3.0:1784948734.587133:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.587138:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:3.0:1784948734.587142:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000063f28b6f. 00000400:00000200:3.0:1784948734.587148:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-5 00000400:00000200:3.0:1784948734.587150:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -5 00000400:00000200:3.0:1784948734.587181:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000400:00000200:3.0:1784948734.587188:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:3.0:1784948734.587192:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.587193:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.587195:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000010:3.0:1784948734.587197:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000400:00000200:3.0:1784948734.587200:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.587203:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:3.0:1784948734.587207:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dc7a25c6. 00000400:00000200:3.0:1784948734.587213:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-5 00000400:00000200:3.0:1784948734.587217:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -5 00000400:00000200:3.0:1784948734.587220:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000400:00000200:3.0:1784948734.587224:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:3.0:1784948734.587227:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.587228:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.587229:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096db69f7 00000400:00000010:3.0:1784948734.587231:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000096db69f7. 00000400:00000200:3.0:1784948734.587234:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.587237:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:3.0:1784948734.587239:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b1cbf66f. 00000400:00000200:3.0:1784948734.587243:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-5 00000400:00000200:3.0:1784948734.587245:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -5 00000400:00000200:3.0:1784948734.587248:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000400:00000200:3.0:1784948734.587255:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:3.0:1784948734.587258:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948734.587260:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948734.587261:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:3.0:1784948734.587263:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000400:00000200:3.0:1784948734.587265:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948734.587267:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:3.0:1784948734.587271:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000208850ab. 00000400:00000200:3.0:1784948734.587277:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-5 00000400:00000200:3.0:1784948734.587281:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -5 00000400:00000200:3.0:1784948734.587284:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000020:00000001:3.0:1784948735.220271:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948735.220277:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948735.220281:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948735.412118:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948735.412135:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948735.412137:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948735.412142:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948735.412146:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948735.412183:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948735.412186:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948735.604037:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948735.604046:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000005e1bb19b. 00000400:00000010:1.0:1784948735.604056:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002c1a3d41. 00000400:00000010:1.0:1784948735.604061:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000010:1.0:1784948735.604070:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d2a46502. 00000400:00000200:1.0:1784948735.604075:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948735.604090:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948735.604097:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948735.604101:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3182:3182:232:4294967295] 192.168.202.137@tcp LPNI seq info [2059:2059:8:4294967295] 00000400:00000200:1.0:1784948735.604109:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948735.604122:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948735.604128:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000100:0.0:1784948735.604132:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000800:00000010:1.0:1784948735.604133:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d104522b. 00000800:00000010:0.0:1784948735.604140:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002429e93e. 00000800:00000200:1.0:1784948735.604157:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784948735.604165:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000200:1.0:1784948735.604171:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:0.0:1784948735.604177:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000800:00000200:1.0:1784948735.604179:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d104522b type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1784948735.604185:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11008 health :700 00000400:00000200:0.0:1784948735.604191:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948735.604195:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:0.0:1784948735.604200:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:3.0:1784948735.604258:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948735.604265:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2861 00000400:00000001:3.0:1784948735.604271:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948735.604272:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948735.604274:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000400:00000010:3.0:1784948735.604276:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000400:00000200:3.0:1784948735.604280:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948735.604283:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:3.0:1784948735.604287:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b8810b40. 00000400:00000200:3.0:1784948735.604292:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:3.0:1784948735.604297:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:3.0:1784948735.604302:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000800:00000001:1.0:1784948735.604313:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948735.604418:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948735.604422:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948735.604426:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948735.604434:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948735.604445:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948735.604454:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948735.604462:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948735.604467:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3183:3183:233:4294967295] 192.168.202.137@tcp LPNI seq info [2060:2060:8:4294967295] 00000400:00000200:1.0:1784948735.604474:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948735.604479:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948735.604483:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000062b4a223. 00000800:00000200:1.0:1784948735.604487:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948735.604492:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948735.604494:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000062b4a223 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:1.0:1784948735.604584:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948735.604590:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000062b4a223. 00000400:00000200:1.0:1784948735.604595:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948735.604602:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948735.604645:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948736.244201:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948736.244207:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948736.244212:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948736.436101:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948736.436118:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948736.436121:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948736.436129:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948736.436133:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948736.436175:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948736.436181:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948737.268249:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948737.268256:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948737.268260:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948737.460183:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948737.460198:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948737.460205:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.460217:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948737.460225:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948737.460230:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948737.460234:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.460262:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948737.460270:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948737.460276:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948737.460288:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948737.460289:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.460296:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.460326:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948737.460334:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948737.460340:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948737.460347:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948737.460356:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948737.460364:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289649024:0@lo:400: 00000100:00000001:0.0:1784948737.460380:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948737.460392:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948737.460398:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948737.460404:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948737.460414:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948737.460424:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948737.460440:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c8dbdfb2. 00000400:00000010:0.0:1784948737.460450:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000100:00000200:0.0:1784948737.460460:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289649024, portal 25 00000100:00000001:0.0:1784948737.460470:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948737.460477:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470022400 : -115544239529216 : ffff96e9c2a5a300) 00000100:00000040:0.0:1784948737.460496:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.460503:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948737.460510:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948737.460516:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948737.460523:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948737.460530:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948737.460539:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948737.460552:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000001:3.0:1784948737.460555:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948737.460561:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948737.460566:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289649024, offset 0 02000000:00000001:3.0:1784948737.460570:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948737.460576:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948737.460582:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948737.460588:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:3.0:1784948737.460592:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000098475f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289649152:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948737.460600:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948737.460605:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948737.460609:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948737.460616:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948737.460617:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:3.0:1784948737.460628:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000b8810b40. 02000000:00000001:3.0:1784948737.460634:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948737.460640:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6434f80 00000400:00000010:3.0:1784948737.460642:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000400:00000010:3.0:1784948737.460655:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000400:00000200:0.0:1784948737.460661:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7751 [2] + 0 00000400:00000010:0.0:1784948737.460669:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005bb236bd. 00000400:00000200:0.0:1784948737.460676:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006799df90 00000100:00000200:3.0:1784948737.460683:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289649152, portal 10 00000400:00000200:0.0:1784948737.460687:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948737.460690:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948737.460696:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470016128 : -115544239535488 : ffff96e9c2a58a80) 00000400:00000200:0.0:1784948737.460706:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:3.0:1784948737.460712:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948737.460723:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006799df90 00000100:00000001:3.0:1784948737.460726:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:0.0:1784948737.460735:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000006799df90. 00000100:00000040:3.0:1784948737.460737:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948737.460746:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 00000100:00000001:0.0:1784948737.460751:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948737.460754:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289649152, offset 0 00000100:00000200:0.0:1784948737.460760:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1784948737.460770:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784948737.460772:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000009b82ab0d x1871651289649024 msgsize 224 00000100:00100000:0.0:1784948737.460789:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784948737.460791:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:0.0:1784948737.460803:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948737.460806:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948737.460816:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948737.460821:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948737.460827:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784948737.460830:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948737.460833:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3184:3184:233:4294967295] 192.168.202.137@tcp LPNI seq info [2061:2061:8:4294967295] 00000400:00000200:3.0:1784948737.460856:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948737.460864:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948737.460871:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000400:00000200:0.0:1784948737.460873:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000400:00000010:0.0:1784948737.460881:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000800:00000200:3.0:1784948737.460884:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948737.460894:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784948737.460905:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948737.460964:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:0.0:1784948737.460972:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.460987:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.460996:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948737.460997:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2032 usecs (3 jiffies) 00000100:00000001:3.0:1784948737.461005:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305676804 00000100:00000040:0.0:1784948737.461011:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948737.461017:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:3.0:1784948737.461031:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948737.461043:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948737.461044:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305676804 00000100:00000001:0.0:1784948737.461055:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:3.0:1784948737.461056:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:3.0:1784948737.461066:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784948737.461074:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948737.461075:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305676804 00000100:00000010:3.0:1784948737.461083:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:0.0:1784948737.461088:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305676843 00000100:00000001:3.0:1784948737.461091:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948737.461103:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305676804 00000100:00000001:0.0:1784948737.461109:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305676804 00000100:00000010:3.0:1784948737.461112:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:3.0:1784948737.461126:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948737.461159:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305676843 00000100:00000001:3.0:1784948737.461164:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948737.461174:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.461175:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305676804 00000100:00000001:3.0:1784948737.461183:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948737.461191:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948737.461194:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305676843 00000100:00000001:3.0:1784948737.461202:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948737.461207:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305676804 00000100:00000001:3.0:1784948737.461209:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948737.461222:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948737.461224:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305676843 00000100:00000001:3.0:1784948737.461229:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948737.461237:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.461240:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305676804 00000100:00000001:3.0:1784948737.461245:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.461257:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948737.461273:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.461286:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.461293:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948737.461307:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948737.461316:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948737.461327:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948737.461334:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948737.461345:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.461361:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948737.461367:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948737.461379:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:3.0:1784948737.461389:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948737.461400:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948737.461407:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096db69f7 00000400:00000010:3.0:1784948737.461410:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000096db69f7. 00000100:00000001:3.0:1784948737.461416:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784948737.461427:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.461456:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948737.461468:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.461521:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948737.461527:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1784948737.461539:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948737.461556:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948737.461564:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948737.461574:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948737.461590:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948737.461613:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435000 00000400:00000200:3.0:1784948737.461630:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 150672 00000800:00000001:3.0:1784948737.461645:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948737.461659:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948737.461669:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948737.461690:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948737.461702:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948737.461710:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:3.0:1784948737.461721:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cca13cd1. 00000100:00000040:3.0:1784948737.461731:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000cca13cd1 x1871651289649152 msgsize 224 00000100:00100000:3.0:1784948737.461750:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784948737.461780:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:3.0:1784948737.461792:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948737.461802:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948737.461844:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948737.461855:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289649152 02000000:00000001:0.0:1784948737.461866:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948737.461874:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948737.461880:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948737.461893:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948737.462019:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289649152 00000020:00000001:0.0:1784948737.462031:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948737.462041:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948737.462051:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948737.462060:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948737.462073:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948737.462086:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948737.462097:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948737.462098:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948737.462105:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:0.0:1784948737.462108:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:0.0:1784948737.462117:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1784948737.462120:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948737.462127:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:0.0:1784948737.462153:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000400:00000010:3.0:1784948737.462156:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c8dbdfb2. 00000020:00000010:0.0:1784948737.462169:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000400:00000200:3.0:1784948737.462170:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:3.0:1784948737.462178:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000020:00000010:0.0:1784948737.462185:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000511be300. 00000100:00000001:3.0:1784948737.462191:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784948737.462198:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000200:3.0:1784948737.462205:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948737.462222:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000200:3.0:1784948737.462227:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948737.462234:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948737.462244:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948737.462255:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.462273:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948737.462275:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948737.462280:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948737.462283:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948737.462292:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784948737.462294:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948737.462300:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948737.462306:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948737.462311:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948737.462320:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.462326:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784948737.462327:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=59 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948737.462343:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=59 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948737.462352:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948737.462355:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948737.462360:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784948737.462361:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00010000:00000001:3.0:1784948737.462365:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.462367:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948737.462371:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948737.462381:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948737.462383:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 668 00000100:00000040:0.0:1784948737.462395:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948737.462403:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948737.462404:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482518016 : -115544227033600 : ffff96e9c3644e00) 00000100:00000001:3.0:1784948737.462412:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948737.462416:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948737.462421:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000cca13cd1 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:398/0 lens 224/0 e 0 to 0 dl 1784948748 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948737.462430:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948737.462463:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:3.0:1784948737.462466:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289649024:0@lo:400: 00000020:00000010:0.0:1784948737.462474:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:3.0:1784948737.462478:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948737.462484:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948737.462485:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948737.462494:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b8226094 x1871651289649024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948737.462498:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00000001:3.0:1784948737.462514:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948737.462520:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:0.0:1784948737.462522:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000cca13cd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289649152:12345-192.168.202.137@tcp:400: 02000000:00000010:3.0:1784948737.462524:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 02000000:00000001:3.0:1784948737.462533:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:0.0:1784948737.462538:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289649152 00000020:00000001:3.0:1784948737.462539:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948737.462544:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948737.462548:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948737.462549:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948737.462555:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000009839174d. 00000020:00000001:0.0:1784948737.462560:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:3.0:1784948737.462561:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948737.462567:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1784948737.462569:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948737.462571:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b8226094. 00000020:00000001:0.0:1784948737.462578:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784948737.462580:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:0.0:1784948737.462585:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:3.0:1784948737.462586:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.462593:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948737.462598:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784948737.462602:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784948737.462604:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.462608:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948737.462610:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784948737.462615:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784948737.462616:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784948737.462620:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948737.462623:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948737.462628:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948737.462633:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948737.462636:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948737.462642:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000005e1d90a1. 02000000:00000001:0.0:1784948737.462646:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.462649:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948737.462651:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948737.462654:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289649152 00010000:00000001:0.0:1784948737.462657:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948737.462663:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000cca13cd1 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:398/0 lens 224/224 e 0 to 0 dl 1784948748 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948737.462674:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948737.462676:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948737.462679:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=54 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948737.462684:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948737.462687:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948737.462690:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948737.462693:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948737.462695:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948737.462697:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948737.462701:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948737.462704:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000200:0.0:1784948737.462708:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289649152, offset 224 00000400:00000200:0.0:1784948737.462713:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948737.462722:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948737.462728:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3185:3185:233:4294967295] 192.168.202.137@tcp LPNI seq info [2062:2062:8:4294967295] 00000400:00000200:0.0:1784948737.462740:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948737.462746:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948737.462751:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002aeee8e9. 00000800:00000200:0.0:1784948737.462756:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948737.462763:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948737.462767:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002aeee8e9 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948737.462788:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948737.462791:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948737.462794:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948737.462796:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948737.462799:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948737.462803:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cca13cd1 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:398/0 lens 224/224 e 0 to 0 dl 1784948748 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948737.462814:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000cca13cd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289649152:12345-192.168.202.137@tcp:400: Request processed in 352us (1082us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948737.462824:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 668 00000100:00000040:0.0:1784948737.462828:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948737.462832:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948737.462834:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948737.462838:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000511be300. 00000020:00000010:0.0:1784948737.462844:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948737.462848:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 00000020:00000040:0.0:1784948737.462853:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948737.462857:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.463238:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948737.463244:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948737.463250:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948737.463253:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948737.463255:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000400:00000200:3.0:1784948737.463264:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:3.0:1784948737.463266:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000100:00000001:3.0:1784948737.463270:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948737.463278:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948737.463291:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.463301:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948737.463303:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.463305:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948737.463308:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948737.463310:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948737.463312:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948737.463314:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948737.463320:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=54 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948737.463324:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=54 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948737.463327:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948737.463328:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948737.463331:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948737.463334:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948737.463352:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948737.463356:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948737.463358:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948737.463360:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948737.463362:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948737.463365:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.463373:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948737.463375:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948737.463377:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948737.463380:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948737.463389:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000098475f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289649152:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948737.463393:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.463395:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948737.463398:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000098475f18 x1871651289649152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948737.463406:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948737.463408:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948737.463411:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000b8810b40. 02000000:00000001:3.0:1784948737.463414:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948737.463416:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948737.463417:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948737.463420:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948737.463422:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000925a7f5a. 02000000:00000001:3.0:1784948737.463425:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948737.463427:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948737.463428:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000098475f18. 00000100:00000001:3.0:1784948737.463431:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948737.463433:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.463435:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948737.463440:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948737.463442:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948737.463444:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948737.463447:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948737.504000:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305676843, at 4305676844 00000100:00000010:0.0:1784948737.504016:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000040:3.0:1784948737.504070:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305676843, at 4305676844 00000100:00000010:3.0:1784948737.504084:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00000100:00000040:1.0:1784948737.504107:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305676843, at 4305676844 00000100:00000010:1.0:1784948737.504120:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000020:00000001:1.0:1784948738.292080:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948738.292084:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948738.292086:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948738.483973:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948738.483984:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948738.483986:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948738.483990:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948738.483992:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948738.483995:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948738.483998:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948739.316203:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948739.316210:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948739.316214:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948739.508029:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948739.508042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948739.508045:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948739.508051:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948739.508054:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948739.508058:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948739.508060:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948740.340188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948740.340195:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948740.340199:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948740.532055:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948740.532068:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948740.532070:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948740.532075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948740.532078:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948740.532090:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948740.532093:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948741.364204:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948741.364210:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948741.364213:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948741.556192:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948741.556203:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784948741.556222:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948716/real 1784948716] req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948741.556247:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948741.556251:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948741.556256:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002eadc14f. 00000400:00000200:1.0:1784948741.556263:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:1.0:1784948741.556267:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000100:00000001:1.0:1784948741.556272:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948741.556278:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948741.556291:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948741.556301:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948741.556305:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.556311:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948741.556313:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948741.556319:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948741.556328:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.556338:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948741.556341:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948741.556345:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948741.556348:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948741.556351:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948741.556354:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948741 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948741.556364:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784948741.556369:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.556373:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948741.556376:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.556379:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784948741.556383:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784948741.556387:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784948741.556423:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:1.0:1784948741.556429:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784948741.556435:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948741 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948741.556446:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289648000:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784948741.556452:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.556456:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948741.556461:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005dccb977 x1871651289648000/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948741 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948741.556472:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948741.556475:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948741.556481:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000e6df9f3d. 02000000:00000001:1.0:1784948741.556488:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948741.556492:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948741.556495:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:1.0:1784948741.556499:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948741.556502:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000900bf136. 02000000:00000001:1.0:1784948741.556508:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948741.556511:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948741.556514:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005dccb977. 00000100:00000001:1.0:1784948741.556519:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948741.556522:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.556526:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.556531:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948741.556533:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948741.556538:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948741.556540:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948741.556542:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948741.556554:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948741.556563:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a191ee5f. 00000020:00000040:0.0:1784948741.556570:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948741.556577:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948741.556580:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948741.556588:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948741.556599:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000006487a147. 00000100:00000001:0.0:1784948741.556608:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948741.556618:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948741.556633:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948741.556635:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948741.556642:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a191ee5f] to pc [ptlrpcd_00_03+3] req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948741.556697:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948741.556717:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948741.556724:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948741.556727:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948741.556729:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1784948741.556732:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784948741.556736:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1784948741.556738:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:1.0:1784948741.556740:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.556746:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1784948741.556752:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1784948741.556758:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10989 00000100:00000001:3.0:1784948741.556765:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948741.556771:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948741.556777:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:3.0:1784948741.556779:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948741.556786:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1784948741.556788:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 10987 00000100:00000001:3.0:1784948741.556791:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784948741.556800:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000040:3.0:1784948741.556804:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948741.556811:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948741.556821:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948741.556829:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948741.556833:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948741.556835:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:3.0:1784948741.556838:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948741.556845:0:89766:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:3.0:1784948741.556848:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1784948741.556850:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:3.0:1784948741.556853:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784948741.556856:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1784948741.556863:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948741.556865:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00100000:3.0:1784948741.556872:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a191ee5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289649280:0@lo:400: 00000020:00000001:0.0:1784948741.556874:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784948741.556882:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:3.0:1784948741.556886:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784948741.556892:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948741.556896:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784948741.556897:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784948741.556899:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 02000000:00000001:3.0:1784948741.556903:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.556904:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:3.0:1784948741.556962:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1784948741.556963:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:0.0:1784948741.556972:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000010:3.0:1784948741.556974:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000b8810b40. 00000100:00080000:0.0:1784948741.556980:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 02000000:00000001:3.0:1784948741.556984:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.556987:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948741.556992:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:3.0:1784948741.556993:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 02000000:00000001:0.0:1784948741.557001:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784948741.557005:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 02000000:00000001:0.0:1784948741.557010:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.557016:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:3.0:1784948741.557016:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289649280, portal 25 00000100:00000001:0.0:1784948741.557020:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.557023:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948741.557028:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558304768 : -115544151246848 : ffff96e9c7e8b800) 00000100:00000010:0.0:1784948741.557032:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000100:00000040:3.0:1784948741.557040:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784948741.557041:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784948741.557054:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784948741.557059:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.557064:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948741.557066:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948741.557071:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1784948741.557073:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784948741.557076:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948741.557081:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 02000000:00000010:0.0:1784948741.557086:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c7c247e1. 00000100:00000200:3.0:1784948741.557089:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289649280, offset 0 00000100:00000001:0.0:1784948741.557122:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948741.557130:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:0.0:1784948741.557135:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c39699a8 x1871651289649408/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948741.557149:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948741.557154:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948741.557156:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:0.0:1784948741.557164:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c39699a8] to pc [ptlrpcd_rcv+-1] req@00000000c39699a8 x1871651289649408/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948741.557178:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435080 00000400:00000200:3.0:1784948741.557191:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7781 [2] + 0 00000100:00000001:0.0:1784948741.557192:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948741.557203:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000529cd956. 00000100:00000040:0.0:1784948741.557204:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948741.557214:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:3.0:1784948741.557216:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f1ae873e 00000100:00000010:0.0:1784948741.557222:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000097a95a31. 00000020:00000040:0.0:1784948741.557229:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:3.0:1784948741.557230:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948741.557235:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948741.557239:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948741.557245:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948741.557246:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948741.557250:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784948741.557259:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000008746b1e6. 00000100:00000001:1.0:1784948741.557260:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784948741.557263:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f1ae873e 00000100:00000001:1.0:1784948741.557267:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948741.557271:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948741.557272:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f1ae873e. 00000100:00000040:1.0:1784948741.557282:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c39699a8 x1871651289649408/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948741.557290:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:0.0:1784948741.557292:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1784948741.557298:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784948741.557311:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000068fd7440 x1871651289649280 msgsize 224 00000100:00000001:1.0:1784948741.557333:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948741.557339:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948741.557342:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:0.0:1784948741.557345:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784948741.557349:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784948741.557352:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948741.557353:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 02000000:00000001:1.0:1784948741.557356:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948741.557365:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948741.557370:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097a95a31] to pc [ptlrpcd_00_00+0] req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948741.557374:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289649408:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784948741.557379:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948741.557390:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784948741.557397:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096db69f7 00000100:00000001:1.0:1784948741.557398:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948741.557405:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000096db69f7. 00000100:00000001:1.0:1784948741.557406:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948741.557418:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.557421:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948741.557422:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.557424:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948741.557433:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784948741.557436:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11018) 00000100:00000200:3.0:1784948741.557439:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948741.557440:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948741.557449:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948741.557454:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948741.557458:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.557466:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948741.557466:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.557475:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948741.557480:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948741.557483:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948741.557490:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948741.557497:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948741.557517:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948741.557521:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948741.557523:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948741.557529:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948741.557534:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948741.557541:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948741.557546:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948741.557547:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.557558:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1784948741.557560:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000097a95a31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289649536:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948741.557568:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948741.557573:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948741.557575:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948741.557576:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948741.557584:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948741.557586:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305680939 02000000:00000010:0.0:1784948741.557591:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 02000000:00000001:0.0:1784948741.557599:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948741.557601:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000400:00000010:0.0:1784948741.557613:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005bb236bd. 00000100:00000001:3.0:1784948741.557617:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000010:0.0:1784948741.557628:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000001:3.0:1784948741.557635:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305680939 00000100:00000200:0.0:1784948741.557639:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289649536, portal 10 00000100:00000010:3.0:1784948741.557646:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:0.0:1784948741.557651:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948741.557655:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784948741.557660:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558302080 : -115544151249536 : ffff96e9c7e8ad80) 00000100:00000001:3.0:1784948741.557664:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305680939 00000100:00000010:3.0:1784948741.557673:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000afdabce. 00000100:00000040:0.0:1784948741.557680:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948741.557684:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948741.557695:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305680939 00000100:00000010:3.0:1784948741.557710:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784948741.557713:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948741.557723:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000040:0.0:1784948741.557725:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948741.557733:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948741.557738:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000100:00000001:3.0:1784948741.557739:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948741.557750:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289649536, offset 0 00000100:00000001:3.0:1784948741.557751:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.557764:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948741.557772:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948741.557791:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948741.557801:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948741.557814:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948741.557823:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948741.557829:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948741.557840:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3186:3186:233:4294967295] 192.168.202.137@tcp LPNI seq info [2063:2063:8:4294967295] 00000400:00000200:0.0:1784948741.557869:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948741.557881:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948741.557891:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002aeee8e9. 00000100:00000001:1.0:1784948741.557939:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948741.557945:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:0.0:1784948741.557959:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948741.557964:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948741.557974:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948741.557983:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948741.557991:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002aeee8e9 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948741.558015:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948741.558024:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2035 usecs (3 jiffies) 00000100:00000001:0.0:1784948741.558038:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305680901 00000100:00000001:0.0:1784948741.558048:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305680939 00000100:00000001:0.0:1784948741.558056:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305680901 00000100:00000001:0.0:1784948741.558068:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305680939 00000100:00000001:0.0:1784948741.558074:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305680901 00000100:00000001:0.0:1784948741.558080:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305680939 00000100:00000001:0.0:1784948741.558087:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305680901 00000100:00000001:0.0:1784948741.558094:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305680939 00000100:00000001:0.0:1784948741.558120:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948741.558124:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.558137:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.558144:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.558160:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948741.558168:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948741.558177:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948741.558182:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948741.558188:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.558195:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948741.558215:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948741.558225:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002aeee8e9. 00000400:00000200:3.0:1784948741.558236:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948741.558241:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948741.558252:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948741.558262:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948741.558263:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:3.0:1784948741.558273:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000800:00000001:1.0:1784948741.558274:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948741.558285:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:1.0:1784948741.558286:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948741.558301:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948741.558310:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948741.558342:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435180 00000100:00000001:3.0:1784948741.558363:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948741.558367:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 150896 00000100:00000040:3.0:1784948741.558376:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948741.558383:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948741.558399:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948741.558409:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948741.558412:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948741.558419:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784948741.558429:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948741.558443:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948741.558448:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948741.558455:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005e43a4d7. 00000100:00000040:1.0:1784948741.558463:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000005e43a4d7 x1871651289649536 msgsize 224 00000100:00100000:1.0:1784948741.558486:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784948741.558511:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:1.0:1784948741.558525:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948741.558535:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948741.558577:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948741.558582:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289649536 02000000:00000001:3.0:1784948741.558585:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948741.558587:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948741.558593:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948741.558604:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948741.558609:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289649536 00000020:00000001:3.0:1784948741.558616:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948741.558623:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784948741.558629:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948741.558637:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948741.558645:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784948741.558656:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948741.558668:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948741.558677:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004562b7bd. 00000100:00000001:3.0:1784948741.558688:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948741.558698:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004562b7bd. 00000020:00000010:3.0:1784948741.558706:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000020:00000010:3.0:1784948741.558712:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013785842. 00000020:00000010:3.0:1784948741.558718:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000040:3.0:1784948741.558730:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948741.558741:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948741.558746:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948741.558755:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948741.558766:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948741.558772:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948741.558777:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.558804:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948741.558832:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948741.558838:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948741.558842:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948741.558846:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948741.558859:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1784948741.558860:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 669 00000100:00000001:1.0:1784948741.558866:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000001:0.0:1784948741.558868:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1784948741.558875:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948741.558876:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:0.0:1784948741.558877:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000100:00000001:1.0:1784948741.558885:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.558887:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451661312 : -115544257890304 : ffff96e9c18d7800) 00000100:00000001:1.0:1784948741.558894:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948741.558895:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000100:00000040:3.0:1784948741.558903:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005e43a4d7 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:402/0 lens 224/0 e 0 to 0 dl 1784948752 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948741.558957:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000100:00000001:0.0:1784948741.558969:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948741.558983:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948741.558990:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948741.559000:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004562b7bd. 00000100:00000001:3.0:1784948741.559010:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948741.559023:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004562b7bd. 00000100:00000200:0.0:1784948741.559029:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948741.559047:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005e43a4d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289649536:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948741.559049:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948741.559054:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948741.559060:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289649536 00000100:00000001:0.0:1784948741.559063:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:3.0:1784948741.559068:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:0.0:1784948741.559070:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:3.0:1784948741.559076:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784948741.559077:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948741.559083:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948741.559084:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948741.559089:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948741.559091:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948741.559123:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00001000:0.0:1784948741.559124:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=63 v=5 (5 5 5 5) 00000020:00000001:3.0:1784948741.559133:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00001000:0.0:1784948741.559135:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=63 v=5 (1 1 1 1) 00000020:00000001:3.0:1784948741.559139:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.559143:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784948741.559149:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784948741.559150:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948741.559155:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00010000:00000001:0.0:1784948741.559157:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784948741.559161:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948741.559163:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948741.559167:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948741.559169:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.559175:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:0.0:1784948741.559180:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948741.559182:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948741.559190:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f1c6c4f3. 02000000:00000001:3.0:1784948741.559197:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.559204:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.559211:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948741.559212:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948741.559221:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289649536 00000100:00000001:0.0:1784948741.559224:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00010000:00000001:3.0:1784948741.559227:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948741.559233:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:3.0:1784948741.559237:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005e43a4d7 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:402/0 lens 224/224 e 0 to 0 dl 1784948752 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948741.559251:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948741.559254:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948741.559260:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948741.559267:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=58 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948741.559280:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948741.559288:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948741.559293:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:0.0:1784948741.559295:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a191ee5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289649280:0@lo:400: 02000000:00000001:3.0:1784948741.559298:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948741.559303:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948741.559311:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948741.559317:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948741.559319:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948741.559325:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 00000100:00000001:0.0:1784948741.559328:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:3.0:1784948741.559332:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289649536, offset 224 00000100:00000040:0.0:1784948741.559341:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a191ee5f x1871651289649280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948741.559347:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948741.559364:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784948741.559372:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:3.0:1784948741.559379:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3187:3187:233:4294967295] 192.168.202.137@tcp LPNI seq info [2064:2064:8:4294967295] 02000000:00000001:0.0:1784948741.559382:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948741.559388:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000b8810b40. 02000000:00000001:0.0:1784948741.559399:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1784948741.559403:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:0.0:1784948741.559405:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948741.559411:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948741.559418:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000800:00000200:3.0:1784948741.559423:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784948741.559424:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000006487a147. 02000000:00000001:0.0:1784948741.559433:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000800:00000010:3.0:1784948741.559435:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 02000000:00000001:0.0:1784948741.559436:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948741.559440:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a191ee5f. 00000100:00000001:0.0:1784948741.559448:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948741.559451:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.559457:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948741.559459:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948741.559465:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948741.559470:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.559473:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.0:1784948741.559479:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:0.0:1784948741.559480:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948741.559492:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948741.559522:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948741.559528:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948741.559533:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948741.559536:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948741.559539:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948741.559546:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005e43a4d7 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:402/0 lens 224/224 e 0 to 0 dl 1784948752 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948741.559559:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005e43a4d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289649536:12345-192.168.202.137@tcp:400: Request processed in 568us (1094us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948741.559571:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 669 00000100:00000040:3.0:1784948741.559576:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948741.559578:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948741.559581:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948741.559586:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013785842. 00000020:00000010:3.0:1784948741.559592:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000020:00000010:3.0:1784948741.559596:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000020:00000040:3.0:1784948741.559601:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948741.559605:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948741.559740:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948741.559755:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948741.559763:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948741.559770:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948741.559787:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948741.559808:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435180 00000400:00000200:3.0:1784948741.559819:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8195 [1] + 224 00000800:00000001:3.0:1784948741.559827:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948741.559839:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948741.559843:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948741.559848:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948741.559855:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948741.559859:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948741.559869:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948741.559890:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948741.559898:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948741.559901:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.560007:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948741.560013:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948741.560019:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948741.560021:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948741.560024:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005bb236bd. 00000400:00000200:0.0:1784948741.560032:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000400:00000010:0.0:1784948741.560034:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000100:00000001:0.0:1784948741.560038:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948741.560046:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948741.560060:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948741.560070:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948741.560072:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.560075:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948741.560077:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948741.560079:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948741.560082:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948741.560084:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948741.560090:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=58 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948741.560094:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=58 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948741.560123:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948741.560125:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948741.560128:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948741.560132:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948741.560141:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.560145:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948741.560147:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948741.560150:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948741.560151:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948741.560155:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948741.560163:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948741.560166:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948741.560168:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948741.560171:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948741.560180:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000097a95a31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289649536:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948741.560186:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.560188:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948741.560190:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000097a95a31 x1871651289649536/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948757 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948741.560198:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948741.560201:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948741.560203:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 02000000:00000001:0.0:1784948741.560207:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948741.560208:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948741.560210:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948741.560214:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948741.560216:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000008746b1e6. 02000000:00000001:0.0:1784948741.560218:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948741.560220:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948741.560222:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000097a95a31. 00000100:00000001:0.0:1784948741.560228:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948741.560230:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.560232:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948741.560237:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948741.560239:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948741.560241:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948741.560243:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948741.600031:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305680939, at 4305680940 00000100:00000010:3.0:1784948741.600046:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:1.0:1784948741.600048:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305680939, at 4305680940 00000100:00000040:0.0:1784948741.600061:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305680939, at 4305680940 00000100:00000010:1.0:1784948741.600064:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:0.0:1784948741.600082:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00000800:00000100:0.0:1784948741.748128:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:0.0:1784948741.748136:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000751c5583. 00000400:00000200:0.0:1784948741.748147:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948741.748156:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948741.748166:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11014 health :900 00000400:00000200:0.0:1784948741.748174:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948741.748181:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:0.0:1784948741.748188:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:3.0:1784948741.748263:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948741.748276:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:3.0:1784948741.748284:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948741.748287:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948741.748290:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003dcd3ddc 00000400:00000010:3.0:1784948741.748294:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003dcd3ddc. 00000400:00000200:3.0:1784948741.748299:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948741.748304:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:3.0:1784948741.748309:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a7af9ab9. 00000400:00000200:3.0:1784948741.748317:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:3.0:1784948741.748320:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:3.0:1784948741.748324:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000020:00000001:1.0:1784948742.388166:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948742.388172:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948742.388176:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1784948742.580290:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948742.580299:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784948742.580308:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000e7e08a5. 00000020:00000040:3.0:1784948742.580316:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948742.580323:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948742.580326:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948742.580331:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948742.580340:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:3.0:1784948742.580348:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948742.580354:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948742.580371:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948742.580374:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948742.580380:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000e7e08a5] to pc [ptlrpcd_00_01+1] req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948742.580414:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948742.580417:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948742.580421:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948742.580424:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784948742.580426:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784948742.580431:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1784948742.580432:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784948742.580437:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 10989 00000100:00000001:0.0:1784948742.580439:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948742.580444:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948742.580452:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948742.580453:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:3.0:1784948742.580463:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11013 00000100:00000001:0.0:1784948742.580481:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948742.580484:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948742.580485:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 02000000:00000001:0.0:1784948742.580491:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784948742.580492:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 02000000:00000001:0.0:1784948742.580496:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1784948742.580497:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1784948742.580503:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948742.580506:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00100000:0.0:1784948742.580511:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000e7e08a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289649664:0@lo:400: 00000100:00000001:3.0:1784948742.580514:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948742.580522:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948742.580523:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948742.580530:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948742.580533:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1784948742.580537:0:89766:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1784948742.580538:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948742.580541:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:0.0:1784948742.580543:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1784948742.580549:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:0.0:1784948742.580553:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 00000020:00000040:3.0:1784948742.580554:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:0.0:1784948742.580562:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948742.580563:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:3.0:1784948742.580569:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:0.0:1784948742.580571:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005bb236bd. 00000100:00000040:3.0:1784948742.580579:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948742.580585:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948742.580587:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000400:00000010:0.0:1784948742.580592:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000001:3.0:1784948742.580592:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:3.0:1784948742.580601:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000200:0.0:1784948742.580603:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289649664, portal 25 00000100:00000001:0.0:1784948742.580609:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1784948742.580612:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000001:0.0:1784948742.580614:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482523392 : -115544227028224 : ffff96e9c3646300) 00000100:00080000:3.0:1784948742.580622:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000040:0.0:1784948742.580626:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948742.580628:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948742.580637:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1784948742.580647:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948742.580654:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784948742.580655:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948742.580662:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:0.0:1784948742.580665:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784948742.580668:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948742.580671:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000100:00000010:3.0:1784948742.580677:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005dccb977. 00000100:00000200:0.0:1784948742.580679:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289649664, offset 0 00000020:00000040:3.0:1784948742.580684:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784948742.580694:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784948742.580699:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948742.580700:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948742.580706:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948742.580710:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948742.580712:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948742.580720:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000077ad1613. 00000400:00000200:0.0:1784948742.580721:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784948742.580728:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948742.580739:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784948742.580745:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435200 00000800:00000200:2.1:1784948742.580755:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:0.0:1784948742.580763:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x77a1 [2] + 0 00000100:00000001:3.0:1784948742.580766:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948742.580771:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948742.580774:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002bff2598. 00000100:00000040:3.0:1784948742.580780:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005dccb977] to pc [ptlrpcd_rcv+-1] req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948742.580784:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d6bfc0b3 00000400:00000200:0.0:1784948742.580795:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948742.580803:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948742.580804:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948742.580811:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1784948742.580814:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6bfc0b3 00000100:00000001:3.0:1784948742.580816:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000010:0.0:1784948742.580818:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d6bfc0b3. 00000100:00000010:3.0:1784948742.580822:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000095d61ef9. 00000020:00000040:3.0:1784948742.580828:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948742.580830:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948742.580831:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1784948742.580836:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:3.0:1784948742.580837:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948742.580841:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948742.580844:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000065a766c3 x1871651289649664 msgsize 224 02000000:00000010:3.0:1784948742.580848:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00100000:0.0:1784948742.580853:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784948742.580855:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948742.580858:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:3.0:1784948742.580863:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948742.580872:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948742.580872:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948742.580874:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948742.580880:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095d61ef9] to pc [ptlrpcd_00_02+2] req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948742.580883:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000400:00000010:0.0:1784948742.580888:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000001:0.0:1784948742.580895:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948742.580902:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948742.580940:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948742.580943:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948758 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948742.580950:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.580953:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948742.580959:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.580961:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.580970:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305681963 00000100:00000040:3.0:1784948742.580970:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11019) 00000100:00000010:0.0:1784948742.580980:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:0.0:1784948742.580990:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948742.580997:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305681963 00000100:00000010:0.0:1784948742.581002:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:3.0:1784948742.581004:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948742.581007:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948742.581009:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948742.581013:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948742.581015:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305681963 00000100:00000040:3.0:1784948742.581020:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784948742.581021:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:0.0:1784948742.581030:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948742.581034:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948742.581037:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581038:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305681963 02000000:00000001:3.0:1784948742.581044:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948742.581047:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1784948742.581049:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 02000000:00000001:3.0:1784948742.581051:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581058:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00100000:3.0:1784948742.581059:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289649792:192.168.202.137@tcp:8: 00000100:00000001:3.0:1784948742.581066:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948742.581067:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948742.581069:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784948742.581071:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948742.581072:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581076:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948742.581077:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948742.581080:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784948742.581082:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784948742.581085:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d8bbc1a8. 00000100:00000001:0.0:1784948742.581110:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1784948742.581112:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289649664 02000000:00000001:3.0:1784948742.581115:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581116:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 02000000:00000001:1.0:1784948742.581121:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948742.581124:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784948742.581127:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000100:00000001:0.0:1784948742.581128:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948742.581128:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948742.581134:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948742.581137:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581139:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948742.581141:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003dcd3ddc. 02000000:00000001:1.0:1784948742.581146:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948742.581151:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289649792, portal 4 00000100:00100000:1.0:1784948742.581155:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289649664 00000100:00000001:3.0:1784948742.581158:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948742.581159:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948742.581163:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948742.581165:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482519808 : -115544227031808 : ffff96e9c3645500) 00000100:00000001:0.0:1784948742.581166:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948742.581169:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784948742.581178:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948742.581184:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000040:3.0:1784948742.581185:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948742.581195:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784948742.581207:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948742.581220:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948742.581222:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:3.0:1784948742.581235:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948742.581244:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000100:00000200:3.0:1784948742.581253:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289649792, offset 0 00000400:00000200:3.0:1784948742.581272:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000010:0.0:1784948742.581273:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:0.0:1784948742.581283:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948742.581288:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000010:0.0:1784948742.581289:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000400:00000200:3.0:1784948742.581301:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1784948742.581302:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000400:00000200:3.0:1784948742.581311:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:0.0:1784948742.581312:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a3c05ade. 00000400:00000200:3.0:1784948742.581320:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1784948742.581323:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:3.0:1784948742.581328:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:0.0:1784948742.581337:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:3.0:1784948742.581339:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3189:3189:233:4294967295] 192.168.202.137@tcp LPNI seq info [2065:2065:8:4294967295] 00000100:00000001:0.0:1784948742.581345:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948742.581350:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948742.581355:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581371:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948742.581371:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948742.581384:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948742.581393:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948742.581401:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000100:00000001:0.0:1784948742.581404:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948742.581411:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948742.581415:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000200:3.0:1784948742.581419:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948742.581421:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948742.581422:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 694 00000100:00000040:0.0:1784948742.581448:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948742.581451:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581456:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770425424 : -115539939126192 : ffff96eac2f89450) 00000800:00000200:3.0:1784948742.581458:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784948742.581460:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948742.581469:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 616 niov 1 nkiov 1 00000100:00000040:0.0:1784948742.581470:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000065a766c3 x1871651289649664/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:403/0 lens 224/0 e 0 to 0 dl 1784948753 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948742.581494:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948742.581496:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948742.581504:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000040:3.0:1784948742.581505:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:0.0:1784948742.581513:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948742.581516:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305681963 00000020:00000010:0.0:1784948742.581524:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000100:00000001:3.0:1784948742.581527:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305681963 00000100:00000001:3.0:1784948742.581533:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305681963 00000100:00000001:3.0:1784948742.581539:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305681963 00000100:00100000:0.0:1784948742.581542:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000065a766c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289649664:12345-0@lo:400: 00000100:00000001:3.0:1784948742.581547:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305681963 00000100:00000001:3.0:1784948742.581557:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305681963 00000100:00000200:0.0:1784948742.581560:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289649664 00000100:00000001:3.0:1784948742.581567:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305681963 00000020:00000001:0.0:1784948742.581569:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784948742.581577:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305681963 00000020:00000001:0.0:1784948742.581578:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948742.581584:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948742.581585:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784948742.581590:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784948742.581596:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948742.581597:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784948742.581607:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948742.581608:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948742.581614:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948742.581616:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948742.581619:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948742.581626:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784948742.581628:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784948742.581630:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948742.581635:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:0.0:1784948742.581638:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948742.581641:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948742.581643:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948742.581645:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784948742.581648:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784948742.581650:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948742.581655:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c7c247e1. 00000800:00000200:1.0:1784948742.581672:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948742.581676:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948742.581678:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784948742.581682:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000100:00000001:0.0:1784948742.581688:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948742.581692:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784948742.581695:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948742.581698:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000040:0.0:1784948742.581705:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289649664 00000400:00000200:1.0:1784948742.581709:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000001:0.0:1784948742.581713:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:3.0:1784948742.581716:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948742.581721:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00010000:00000200:0.0:1784948742.581723:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000065a766c3 x1871651289649664/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:403/0 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948742.581725:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784948742.581730:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000800:00000001:3.0:1784948742.581734:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948742.581743:0:337112:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:0.0:1784948742.581753:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:3.0:1784948742.581753:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:1.0:1784948742.581758:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948742.581759:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948742.581767:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=64 v=5 (1 1 1 1) 00000400:00000200:3.0:1784948742.581778:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6435280 00000100:00000001:0.0:1784948742.581783:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948742.581791:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948742.581798:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000100:3.0:1784948742.581800:0:337114:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289649792 offset 0 length 520: 4 00000100:00000001:1.0:1784948742.581804:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784948742.581805:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:3.0:1784948742.581807:0:337114:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000001a085fa2 is in inconsistent state, don't perform health checking (-2, 0) 02000000:00000001:0.0:1784948742.581809:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948742.581812:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000040:1.0:1784948742.581814:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948767 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948742.581818:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:3.0:1784948742.581826:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784948742.581827:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948742.581835:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000100:00000200:0.0:1784948742.581841:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289649664, offset 224 00000100:00000001:1.0:1784948742.581841:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948742.581850:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948742.581856:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:1.0:1784948742.581867:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948742.581869:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948742.581878:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435200 00000800:00000001:1.0:1784948742.581880:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948742.581887:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x81ad [1] + 224 00000800:00000001:1.0:1784948742.581887:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948742.581896:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948742.581901:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948742.581912:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948742.581917:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948742.581919:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948742.581925:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948742.581935:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000e7e08a5 x1871651289649664/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948742.581949:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948742.581960:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948742.581967:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948742.581977:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435300 00000400:00000200:1.0:1784948742.582002:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 151120 00000800:00000001:1.0:1784948742.582024:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948742.582028:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948742.582042:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000800:00000001:1.0:1784948742.582045:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948742.582048:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000400:00000200:1.0:1784948742.582054:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948742.582057:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948742.582060:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948742.582067:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948742.582069:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948742.582075:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784948742.582077:0:337112:0:(events.c:286:request_in_callback()) Process entered 00010000:00000001:0.0:1784948742.582078:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948742.582085:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948742.582085:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:0.0:1784948742.582110:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948742.582121:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000065a766c3 x1871651289649664/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:403/0 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784948742.582122:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000941b125. 00000100:00000040:1.0:1784948742.582133:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000000941b125 x1871651289649920 msgsize 224 00000100:00100000:0.0:1784948742.582154:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000065a766c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289649664:12345-0@lo:400: Request processed in 654us (1304us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948742.582159:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00100000:0.0:1784948742.582180:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 694 00000100:00000001:1.0:1784948742.582184:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784948742.582187:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948742.582195:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000001:1.0:1784948742.582198:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948742.582201:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:1.0:1784948742.582208:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948742.582210:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a3c05ade. 00000020:00000010:0.0:1784948742.582222:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948742.582233:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000100:00000001:3.0:1784948742.582236:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948742.582244:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289649920 00000020:00000040:0.0:1784948742.582247:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:3.0:1784948742.582253:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:0.0:1784948742.582257:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948742.582259:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000010:0.0:1784948742.582263:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c7c247e1. 00000100:00000001:3.0:1784948742.582265:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948742.582268:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948742.582276:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948742.582276:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948742.582287:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:3.0:1784948742.582287:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289649920 00000020:00000001:3.0:1784948742.582293:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:0.0:1784948742.582297:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002bff2598. 00000020:00000040:3.0:1784948742.582298:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784948742.582305:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784948742.582307:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d4d2d632. 00000020:00000040:3.0:1784948742.582311:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948742.582319:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784948742.582323:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948742.582331:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948742.582338:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ea5f5298. 00000100:00000001:3.0:1784948742.582344:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948742.582352:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ea5f5298. 00000020:00000010:3.0:1784948742.582363:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000020:00000010:3.0:1784948742.582371:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013785842. 00000020:00000010:3.0:1784948742.582379:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000040:3.0:1784948742.582390:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948742.582397:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948742.582401:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948742.582406:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948742.582413:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948742.582418:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948742.582422:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948742.582439:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948742.582453:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948742.582456:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948742.582465:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 670 00000100:00000040:3.0:1784948742.582471:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948742.582476:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451658624 : -115544257892992 : ffff96e9c18d6d80) 00000100:00000040:3.0:1784948742.582487:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000941b125 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:403/0 lens 224/0 e 0 to 0 dl 1784948753 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948742.582496:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948742.582499:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ea5f5298. 00000100:00000001:3.0:1784948742.582501:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948742.582504:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ea5f5298. 00000100:00100000:3.0:1784948742.582508:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000941b125 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289649920:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784948742.582511:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289649920 00000020:00000001:3.0:1784948742.582513:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948742.582516:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948742.582518:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948742.582520:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948742.582522:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784948742.582524:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948742.582526:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948742.582528:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948742.582530:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948742.582531:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948742.582533:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948742.582536:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948742.582538:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948742.582541:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002fd89d29. 02000000:00000001:3.0:1784948742.582543:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948742.582545:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948742.582547:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948742.582549:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289649920 00010000:00000001:3.0:1784948742.582551:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948742.582554:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000941b125 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:403/0 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948742.582560:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948742.582562:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948742.582564:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=59 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948742.582567:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948742.582569:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948742.582570:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784948742.582573:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948742.582574:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948742.582576:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948742.582578:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948742.582581:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000100:00000200:3.0:1784948742.582584:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289649920, offset 224 00000400:00000200:3.0:1784948742.582588:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948742.582594:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948742.582598:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3191:3191:233:4294967295] 192.168.202.137@tcp LPNI seq info [2067:2067:8:4294967295] 00000400:00000200:3.0:1784948742.582608:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948742.582612:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948742.582615:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784948742.582618:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948742.582623:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948742.582626:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948742.582640:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948742.582643:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948742.582645:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948742.582647:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948742.582648:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948742.582652:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000941b125 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:403/0 lens 224/224 e 0 to 0 dl 1784948753 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948742.582662:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000941b125 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289649920:12345-192.168.202.137@tcp:400: Request processed in 163us (526us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948742.582671:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 670 00000100:00000040:3.0:1784948742.582674:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948742.582676:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948742.582678:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948742.582680:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013785842. 00000020:00000010:3.0:1784948742.582684:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000020:00000010:3.0:1784948742.582688:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000020:00000040:3.0:1784948742.582691:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948742.582693:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948742.582725:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948742.582729:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:0.0:1784948742.582734:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948742.582740:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948742.582745:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000400:00000010:0.0:1784948742.582747:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 00000100:00000001:0.0:1784948742.582750:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948742.582752:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948742.582890:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948742.582894:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948742.582898:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948742.582900:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948742.582902:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000086a2279a. 00000400:00000200:1.0:1784948742.582924:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:1.0:1784948742.582926:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000100:00000001:1.0:1784948742.582929:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948742.582933:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948742.582942:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948742.582949:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948742.582950:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948742.582952:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948742.582955:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948742.582956:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948742.582958:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948742.582960:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948742.582964:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=59 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948742.582967:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=59 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948742.582970:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948742.582971:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948742.582973:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948742.582976:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948742.582982:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948742.582984:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948742.582985:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948742.582987:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948742.582988:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948742.582990:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948742.582996:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948742.582998:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948742.582999:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948742.583002:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948742.583008:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000095d61ef9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289649920:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948742.583012:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948742.583013:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948742.583015:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000095d61ef9 x1871651289649920/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948742.583020:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948742.583022:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948742.583023:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:1.0:1784948742.583026:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948742.583028:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948742.583030:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948742.583032:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948742.583033:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:1.0:1784948742.583036:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948742.583037:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948742.583038:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000095d61ef9. 00000100:00000001:1.0:1784948742.583041:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948742.583042:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948742.583044:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948742.583047:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948742.583049:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948742.583051:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948742.583053:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948742.623969:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305681963, at 4305681964 00000100:00000010:0.0:1784948742.623981:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:1.0:1784948742.623991:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305681963, at 4305681964 00000100:00000040:3.0:1784948742.623991:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305681963, at 4305681964 00000100:00000010:1.0:1784948742.624002:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000010:3.0:1784948742.624003:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000400:00000100:1.0:1784948743.092009:0:344008:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 1.2.3.4:988 00000400:00000100:1.0:1784948743.092056:0:344008:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 1.2.3.4@tcp at host 1.2.3.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948743.092064:0:344008:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 1.2.3.4@tcp: down 00000400:00000200:1.0:1784948743.092145:0:344008:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784948743.092150:0:344008:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:1.0:1784948743.092153:0:344008:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007b2ad4bf. 00000400:00000200:1.0:1784948743.092157:0:344008:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948743.092161:0:344008:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948743.092165:0:344008:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11016 health :900 00000400:00000200:1.0:1784948743.092170:0:344008:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948743.092172:0:344008:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:1.0:1784948743.092176:0:344008:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000020:00000001:3.0:1784948743.412140:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948743.412144:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948743.412147:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948744.436207:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948744.436212:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948744.436217:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:1.0:1784948744.819994:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948744.820002:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000007367a1f2. 00000400:00000010:1.0:1784948744.820011:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b105ef1a. 00000400:00000010:1.0:1784948744.820017:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000400:00000010:1.0:1784948744.820024:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000e71a71c. 00000400:00000200:1.0:1784948744.820028:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948744.820052:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948744.820057:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948744.820061:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3192:3192:234:4294967295] 192.168.202.137@tcp LPNI seq info [2068:2068:8:4294967295] 00000400:00000200:1.0:1784948744.820067:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948744.820090:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948744.820095:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948744.820127:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000062b4a223. 00000800:00000200:1.0:1784948744.820135:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948744.820147:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948744.820153:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000062b4a223 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948744.820184:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11018 health :900 00000800:00000001:0.0:1784948744.820404:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948744.820416:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948744.820422:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948744.820432:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948744.820444:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948744.820458:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948744.820472:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948744.820488:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948744.820492:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3193:3193:234:4294967295] 192.168.202.137@tcp LPNI seq info [2069:2069:8:4294967295] 00000400:00000200:0.0:1784948744.820502:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948744.820505:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948744.820509:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008746b1e6. 00000800:00000200:0.0:1784948744.820512:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948744.820516:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948744.820518:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008746b1e6 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:0.0:1784948744.820626:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948744.820643:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008746b1e6. 00000400:00000200:0.0:1784948744.820653:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948744.820677:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000020:00000001:3.0:1784948745.460111:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948745.460117:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948745.460121:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1784948746.484207:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948746.484214:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948746.484217:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1784948747.188030:0:344014:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.9:988 00000400:00000100:1.0:1784948747.188032:0:344013:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.7:988 00000400:00000100:0.0:1784948747.188033:0:344011:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.3:988 00000400:00000100:0.0:1784948747.188126:0:344011:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.3@tcp at host 111.222.142.3:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784948747.188126:0:344013:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.7@tcp at host 111.222.142.7:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784948747.188126:0:344014:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.9@tcp at host 111.222.142.9:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948747.188144:0:344013:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.7@tcp: down 00000400:00000200:3.0:1784948747.188144:0:344014:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.9@tcp: down 00000400:00000200:0.0:1784948747.188148:0:344011:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.3@tcp: down 00000400:00000200:3.0:1784948747.188153:0:344014:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784948747.188160:0:344013:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1784948747.188160:0:344014:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000800:00000010:3.0:1784948747.188165:0:344014:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004805f207. 00000400:00000200:0.0:1784948747.188166:0:344011:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784948747.188168:0:344013:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.7@tcp 00000800:00000010:1.0:1784948747.188172:0:344013:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a219b995. 00000400:00000200:3.0:1784948747.188173:0:344014:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:0.0:1784948747.188177:0:344011:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.3@tcp 00000400:00000200:1.0:1784948747.188180:0:344013:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000010:0.0:1784948747.188184:0:344011:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004f41fdbc. 00000400:00000200:3.0:1784948747.188185:0:344014:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948747.188192:0:344011:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948747.188193:0:344013:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948747.188197:0:344014:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11020 health :900 00000400:00000200:0.0:1784948747.188202:0:344011:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948747.188205:0:344014:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948747.188206:0:344013:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948747.188207:0:344011:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948747.188214:0:344014:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:1.0:1784948747.188215:0:344013:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:0.0:1784948747.188216:0:344011:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:3.0:1784948747.188224:0:344014:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000200:1.0:1784948747.188233:0:344013:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:0.0:1784948747.188242:0:344011:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000100:3.0:1784948747.188283:0:344010:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.1:988 00000400:00000100:3.0:1784948747.188308:0:344010:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.1@tcp at host 111.222.142.1:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948747.188315:0:344010:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.1@tcp: down 00000400:00000200:3.0:1784948747.188330:0:344010:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1784948747.188337:0:344010:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000800:00000010:3.0:1784948747.188341:0:344010:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e3ed36ac. 00000400:00000200:3.0:1784948747.188346:0:344010:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948747.188354:0:344010:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948747.188360:0:344010:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948747.188367:0:344010:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:3.0:1784948747.188376:0:344010:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000100:3.0:1784948747.188395:0:344012:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.5:988 00000400:00000100:3.0:1784948747.188415:0:344012:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.5@tcp at host 111.222.142.5:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948747.188424:0:344012:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.5@tcp: down 00000400:00000200:3.0:1784948747.188430:0:344012:0:(router.c:1888:lnet_notify()) Old news 00000020:00000001:3.0:1784948747.508041:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948747.508045:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948747.508060:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948747.700035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948747.700071:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948747.700073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948747.700077:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948747.700078:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948747.700082:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948747.700084:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948747.700085:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.700088:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948747.700093:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000dc33132b. 00000020:00000040:1.0:1784948747.700099:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948747.700103:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948747.700106:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948747.700111:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948747.700118:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00000001:1.0:1784948747.700124:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948747.700132:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000dc33132b x1871651289650048/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948747.700140:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948747.700143:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948747.700147:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc33132b] to pc [ptlrpcd_00_03+3] req@00000000dc33132b x1871651289650048/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948747.700172:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948747.700174:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948747.700178:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948747.700180:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948747.700183:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948747.700187:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e8f14423. 00000100:00000001:0.0:1784948747.700190:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948747.700194:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948747.700196:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948747.700201:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948747.700203:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948747.700208:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948747.700214:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003a8a98bc. 00000100:00000001:1.0:1784948747.700218:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948747.700225:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948747.700235:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948747.700237:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948747.700244:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e8f14423] to pc [ptlrpcd_00_00+0] req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948747.700286:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948747.700292:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11024) 00000100:00000001:0.0:1784948747.700295:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948747.700299:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948747.700301:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948747.700308:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948747.700317:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948747.700320:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948747.700320:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.700323:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948747.700327:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948747.700329:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948747.700331:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948747.700337:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289650176:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948747.700342:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948747.700346:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948747.700349:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948747.700353:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948747.700357:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eff166f6. 02000000:00000001:0.0:1784948747.700361:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948747.700367:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001c023af. 00000400:00000010:0.0:1784948747.700382:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000100:00000200:0.0:1784948747.700388:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289650176, portal 10 00000100:00000001:0.0:1784948747.700392:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948747.700393:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451655040 : -115544257896576 : ffff96e9c18d5f80) 00000100:00000040:0.0:1784948747.700400:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948763 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948747.700414:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948747.700419:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948747.700423:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00000100:00000200:0.0:1784948747.700426:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289650176, offset 0 00000400:00000200:0.0:1784948747.700433:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948747.700445:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948747.700450:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948747.700453:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948747.700458:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948747.700461:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948747.700464:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3194:3194:239:4294967295] 192.168.202.137@tcp LPNI seq info [2070:2070:8:4294967295] 00000400:00000200:0.0:1784948747.700472:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948747.700477:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948747.700481:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008746b1e6. 00000800:00000200:0.0:1784948747.700488:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948747.700496:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948747.700502:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008746b1e6 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948747.700522:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948747.700528:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2037 usecs (3 jiffies) 00000100:00000001:0.0:1784948747.700534:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305687043 00000100:00000010:0.0:1784948747.700540:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:0.0:1784948747.700548:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948747.700555:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305687043 00000100:00000010:0.0:1784948747.700559:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:0.0:1784948747.700565:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784948747.700577:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305687043 00000100:00000010:0.0:1784948747.700584:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:0.0:1784948747.700590:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948747.700596:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305687043 00000100:00000010:0.0:1784948747.700600:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:0.0:1784948747.700605:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948747.700611:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948747.700614:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.700617:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.700620:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948747.700627:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948747.700631:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948747.700635:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784948747.700636:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948747.700639:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1784948747.700641:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650048 00000100:00000001:0.0:1784948747.700642:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948747.700644:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948747.700646:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948747.700648:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948747.700650:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948747.700654:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948747.700657:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650048 00000020:00000001:3.0:1784948747.700659:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948747.700661:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784948747.700664:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948747.700667:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:3.0:1784948747.700669:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784948747.700671:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948747.700675:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948747.700678:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d30e1e5e. 00000100:00000001:3.0:1784948747.700681:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948747.700684:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d30e1e5e. 00000020:00000010:3.0:1784948747.700689:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000020:00000010:3.0:1784948747.700693:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013785842. 00000020:00000010:3.0:1784948747.700698:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000040:3.0:1784948747.700705:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948747.700708:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948747.700710:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948747.700711:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948747.700718:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948747.700728:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948747.700736:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948747.700738:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948747.700743:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 695 00000100:00000040:3.0:1784948747.700746:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948747.700748:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770428496 : -115539939123120 : ffff96eac2f8a050) 00000100:00000040:3.0:1784948747.700756:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009c11a764 x1871651289650048/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:408/0 lens 224/0 e 0 to 0 dl 1784948758 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948747.700763:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948747.700775:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948747.700780:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948747.700785:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948747.700785:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948747.700790:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d30e1e5e. 00000100:00000001:3.0:1784948747.700794:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948747.700797:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000010:3.0:1784948747.700800:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d30e1e5e. 00000100:00100000:3.0:1784948747.700809:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009c11a764 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289650048:12345-0@lo:400: 00000400:00000200:1.0:1784948747.700812:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435400 00000100:00000200:3.0:1784948747.700820:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650048 00000400:00000200:1.0:1784948747.700825:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 151344 00000020:00000001:3.0:1784948747.700825:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948747.700831:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000001:1.0:1784948747.700833:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948747.700834:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948747.700839:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:1.0:1784948747.700840:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948747.700842:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:1.0:1784948747.700846:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784948747.700847:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948747.700851:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948747.700856:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784948747.700856:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948747.700859:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948747.700862:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:3.0:1784948747.700862:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948747.700865:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000200:1.0:1784948747.700866:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784948747.700868:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000010:1.0:1784948747.700870:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b55e20f2. 02000000:00000001:3.0:1784948747.700872:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:1.0:1784948747.700874:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000b55e20f2 x1871651289650176 msgsize 224 02000000:00000010:3.0:1784948747.700876:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000011ff6ec2. 00000100:00100000:1.0:1784948747.700882:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:3.0:1784948747.700882:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948747.700886:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948747.700888:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948747.700890:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650048 00010000:00000001:3.0:1784948747.700893:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784948747.700897:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000200:3.0:1784948747.700897:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009c11a764 x1871651289650048/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:408/0 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948747.700926:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948747.700929:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:3.0:1784948747.700929:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948747.700933:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650176 00000100:00001000:3.0:1784948747.700935:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=69 v=5 (1 1 1 1) 02000000:00000001:0.0:1784948747.700936:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948747.700939:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000001:1.0:1784948747.700940:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948747.700941:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948747.700942:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948747.700945:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948747.700946:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948747.700946:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00100000:0.0:1784948747.700953:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650176 00000100:00000001:3.0:1784948747.700953:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:0.0:1784948747.700957:0:342762:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:3.0:1784948747.700958:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000040:0.0:1784948747.700959:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000001:3.0:1784948747.700961:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948747.700962:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948747.700965:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1784948747.700966:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948747.700970:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:3.0:1784948747.700970:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:0.0:1784948747.700974:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000010:3.0:1784948747.700976:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000100:00000001:0.0:1784948747.700978:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:3.0:1784948747.700981:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289650048, offset 224 00000020:00000010:0.0:1784948747.700982:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000154da45a. 00000100:00000001:0.0:1784948747.700986:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948747.700990:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:0.0:1784948747.700991:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000154da45a. 00000020:00000010:0.0:1784948747.700997:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000400:00000200:3.0:1784948747.701001:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784948747.701003:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a3c05ade. 00000020:00000010:0.0:1784948747.701008:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:3.0:1784948747.701010:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435380 00000400:00000200:3.0:1784948747.701016:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x81d5 [1] + 224 00000100:00000040:0.0:1784948747.701017:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948747.701021:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:3.0:1784948747.701021:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948747.701023:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:3.0:1784948747.701026:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948747.701027:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948747.701030:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948747.701033:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:3.0:1784948747.701034:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000dc33132b x1871651289650048/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948747.701036:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948747.701040:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.701063:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948747.701063:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000dc33132b x1871651289650048/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948747.701076:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948747.701078:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948747.701081:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:0.0:1784948747.701085:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 671 00000100:00000040:0.0:1784948747.701087:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948747.701090:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451651456 : -115544257900160 : ffff96e9c18d5180) 00000400:00000200:3.0:1784948747.701090:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:3.0:1784948747.701094:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000100:00000040:0.0:1784948747.701096:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b55e20f2 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:408/0 lens 224/0 e 0 to 0 dl 1784948758 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948747.701098:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948747.701102:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784948747.701107:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.701109:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948747.701114:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000154da45a. 00000100:00000040:3.0:1784948747.701114:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948747.701121:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784948747.701121:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948747.701124:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948747.701126:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000154da45a. 00000020:00000001:3.0:1784948747.701128:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948747.701134:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009c11a764 x1871651289650048/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:408/0 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948747.701135:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b55e20f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289650176:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948747.701145:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650176 00000020:00000001:0.0:1784948747.701150:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948747.701154:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948747.701157:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948747.701162:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00100000:3.0:1784948747.701164:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009c11a764 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289650048:12345-0@lo:400: Request processed in 374us (663us total) trans 0 rc 0/0 00000020:00000001:0.0:1784948747.701166:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948747.701172:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948747.701176:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948747.701180:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00100000:3.0:1784948747.701181:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 695 00000020:00000001:0.0:1784948747.701183:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948747.701186:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948747.701188:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:0.0:1784948747.701190:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948747.701194:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948747.701196:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948747.701197:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:0.0:1784948747.701199:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:3.0:1784948747.701203:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013785842. 02000000:00000010:0.0:1784948747.701204:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c9ef0f03. 00000020:00000010:3.0:1784948747.701207:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 02000000:00000001:0.0:1784948747.701208:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.701213:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948747.701213:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000020:00000001:0.0:1784948747.701218:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948747.701220:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000040:0.0:1784948747.701221:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650176 00010000:00000001:0.0:1784948747.701226:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:3.0:1784948747.701227:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948747.701231:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000011ff6ec2. 00010000:00000200:0.0:1784948747.701234:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b55e20f2 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:408/0 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948747.701235:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948747.701238:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948747.701244:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:0.0:1784948747.701251:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000010:3.0:1784948747.701251:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000382215bc. 00010000:00000001:0.0:1784948747.701254:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948747.701257:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000048b052c7. 00000100:00001000:0.0:1784948747.701260:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=64 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948747.701266:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948747.701270:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948747.701273:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948747.701275:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948747.701278:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.701280:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948747.701285:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948747.701291:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00000200:0.0:1784948747.701296:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289650176, offset 224 00000400:00000200:0.0:1784948747.701303:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948747.701312:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948747.701322:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3195:3195:239:4294967295] 192.168.202.137@tcp LPNI seq info [2071:2071:8:4294967295] 00000400:00000200:0.0:1784948747.701339:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948747.701347:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948747.701353:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006487a147. 00000800:00000200:0.0:1784948747.701358:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948747.701366:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948747.701370:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000006487a147 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948747.701385:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948747.701393:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948747.701396:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948747.701398:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948747.701400:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948747.701403:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b55e20f2 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:408/0 lens 224/224 e 0 to 0 dl 1784948758 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948747.701410:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b55e20f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289650176:12345-192.168.202.137@tcp:400: Request processed in 300us (535us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948747.701416:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 671 00000100:00000040:0.0:1784948747.701418:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948747.701420:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948747.701421:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948747.701423:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a3c05ade. 00000020:00000010:0.0:1784948747.701426:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948747.701428:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 00000020:00000040:0.0:1784948747.701430:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948747.701432:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948747.701467:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948747.701470:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006487a147. 00000400:00000200:3.0:1784948747.701473:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948747.701478:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948747.701483:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:3.0:1784948747.701485:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000100:00000001:3.0:1784948747.701487:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948747.701491:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948747.701570:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948747.701573:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948747.701576:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948747.701577:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948747.701578:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000001c023af. 00000400:00000200:0.0:1784948747.701581:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000400:00000010:0.0:1784948747.701582:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000001:0.0:1784948747.701584:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948747.701587:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948747.701593:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948747.701597:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948747.701598:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948747.701599:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948747.701601:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948747.701602:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948747.701603:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948747.701605:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948747.701607:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=64 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948747.701610:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=64 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948747.701611:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948747.701612:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948747.701613:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948747.701615:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948747.701620:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.701622:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948747.701623:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948747.701624:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948747.701624:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948747.701626:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948747.701630:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948747.701632:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948747.701633:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948747.701635:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948747.701639:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289650176:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948747.701641:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948747.701642:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948747.701644:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e8f14423 x1871651289650176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948747.701647:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948747.701648:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948747.701649:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eff166f6. 02000000:00000001:0.0:1784948747.701651:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948747.701651:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948747.701653:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948747.701654:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948747.701655:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003a8a98bc. 02000000:00000001:0.0:1784948747.701657:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948747.701658:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948747.701659:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e8f14423. 00000100:00000001:0.0:1784948747.701661:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948747.701662:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948747.701663:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948747.701665:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948747.701666:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948747.701667:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948747.701668:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948747.742971:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305687082, at 4305687083 00000100:00000040:1.0:1784948747.742972:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305687082, at 4305687083 00000100:00000040:0.0:1784948747.742978:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305687082, at 4305687083 00000100:00000010:1.0:1784948747.742987:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:3.0:1784948747.742989:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000010:0.0:1784948747.742994:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000020:00000001:1.0:1784948748.532215:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948748.532222:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948748.532226:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948748.723992:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948748.724005:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948748.724007:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948748.724012:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948748.724014:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948748.724017:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948748.724018:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948748.916144:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784948748.916159:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000ff2aa94d. 00000400:00000010:3.0:1784948748.916172:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000208850ab. 00000400:00000010:3.0:1784948748.916180:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000400:00000010:3.0:1784948748.916204:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003e3841d6. 00000400:00000200:3.0:1784948748.916210:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948748.916236:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948748.916244:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948748.916249:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3196:3196:239:4294967295] 192.168.202.137@tcp LPNI seq info [2072:2072:8:4294967295] 00000400:00000200:3.0:1784948748.916257:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948748.916267:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948748.916274:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948748.916280:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000800:00000200:3.0:1784948748.916288:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948748.916299:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948748.916305:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784948748.916328:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11022 health :900 00000800:00000200:1.0:1784948748.916516:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948748.916524:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:1.0:1784948748.916535:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948748.916543:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784948748.916551:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1784948748.916555:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948748.916570:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948748.916583:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948748.916587:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948748.916594:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948748.916602:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948748.916618:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948748.916631:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948748.916643:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948748.916648:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3197:3197:239:4294967295] 192.168.202.137@tcp LPNI seq info [2073:2073:8:4294967295] 00000400:00000200:1.0:1784948748.916660:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948748.916666:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948748.916670:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008f5595bf. 00000800:00000200:1.0:1784948748.916675:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948748.916683:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948748.916688:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f5595bf type 3, nob 144 niov 1 nkiov 1 00000800:00000200:1.0:1784948748.916847:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948748.916853:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008f5595bf. 00000800:00000001:3.0:1784948748.916856:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948748.916859:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948748.916869:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948748.916874:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948748.916880:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948748.916896:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948748.916943:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948748.916962:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x81f1 00000800:00000001:3.0:1784948748.916970:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948748.916981:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948748.916984:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948748.916989:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948748.916995:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:3.0:1784948748.916999:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000400:00000200:3.0:1784948748.917002:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948748.917006:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000ff2aa94d. 00000400:00000010:3.0:1784948748.917010:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000208850ab. 00000800:00000001:3.0:1784948748.917019:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948748.917022:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948749.556235:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948749.556241:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948749.556244:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948749.748085:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948749.748097:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948749.748099:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948749.748104:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948749.748107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948749.748110:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948749.748111:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948750.580189:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948750.580194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948750.580198:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948750.772012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948750.772069:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948750.772071:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948750.772076:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948750.772080:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948750.772083:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948750.772086:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948751.604203:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948751.604209:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948751.604213:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948751.796055:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948751.796078:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948751.796082:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948751.796092:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948751.796096:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948751.796102:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948751.796107:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948752.628140:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948752.628146:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948752.628150:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948752.820034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1784948752.820067:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948752.820069:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948752.820075:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948752.820084:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948752.820087:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948752.820090:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948752.820096:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948752.820098:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000097a95a31. 00000100:00000001:1.0:1784948752.820100:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948752.820111:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948752.820120:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948752.820126:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948752.820134:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948752.820147:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000eafb2cdc. 00000100:00000001:0.0:1784948752.820159:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948752.820173:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948752.820194:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948752.820199:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948752.820208:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097a95a31] to pc [ptlrpcd_00_01+1] req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948752.820262:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948752.820267:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948752.820274:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948752.820279:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948752.820286:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948752.820292:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a191ee5f. 00000100:00000001:1.0:1784948752.820297:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784948752.820299:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948752.820302:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948752.820308:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784948752.820309:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784948752.820318:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1784948752.820324:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948752.820325:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948752.820340:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000086e4d21. 00000100:00000001:1.0:1784948752.820354:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948752.820355:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948752.820357:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948752.820368:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948752.820372:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948752.820376:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1784948752.820379:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948752.820388:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000097a95a31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289650304:0@lo:400: 00000100:00000001:1.0:1784948752.820402:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948752.820410:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784948752.820412:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784948752.820416:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948752.820418:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948752.820422:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948752.820433:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006620b0a1. 00000100:00000040:0.0:1784948752.820434:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a191ee5f] to pc [ptlrpcd_00_02+2] req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948752.820438:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948752.820450:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eadc14f. 00000400:00000010:1.0:1784948752.820461:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007d821c01. 00000100:00000200:1.0:1784948752.820469:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289650304, portal 25 00000100:00000001:1.0:1784948752.820477:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948752.820482:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558302080 : -115544151249536 : ffff96e9c7e8ad80) 00000100:00000040:1.0:1784948752.820490:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948752.820500:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948752.820504:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948752.820509:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784948752.820513:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289650304, offset 0 00000100:00000001:0.0:1784948752.820514:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948752.820525:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:0.0:1784948752.820527:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11029) 00000400:00000200:1.0:1784948752.820548:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948752.820554:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948752.820562:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948752.820575:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435480 00000400:00000200:1.0:1784948752.820594:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x77d5 [2] + 0 00000400:00000010:1.0:1784948752.820603:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000eed58f44. 00000400:00000200:1.0:1784948752.820610:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000004089dc8d 00000400:00000200:1.0:1784948752.820624:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948752.820641:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948752.820649:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004089dc8d 00000400:00000010:1.0:1784948752.820657:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000004089dc8d. 00000100:00000001:1.0:1784948752.820670:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948752.820677:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784948752.820685:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000001e461ac8 x1871651289650304 msgsize 224 00000100:00100000:1.0:1784948752.820700:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784948752.820706:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948752.820729:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948752.820743:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784948752.820752:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:1.0:1784948752.820761:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948752.820769:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:1.0:1784948752.820772:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948752.820776:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650304 00000100:00000001:1.0:1784948752.820795:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948752.820803:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948752.820814:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948752.820819:0:337120:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1784948752.820837:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948752.820843:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948752.820849:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948752.820858:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948752.820865:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650304 00000020:00000001:3.0:1784948752.820873:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948752.820878:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784948752.820885:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948752.820893:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:3.0:1784948752.820901:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784948752.820957:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948752.820964:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948752.820987:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948752.820996:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305692203 00000100:00000010:1.0:1784948752.821032:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:1.0:1784948752.821045:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784948752.821060:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305692203 00000100:00000010:1.0:1784948752.821072:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000001:1.0:1784948752.821078:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:3.0:1784948752.821083:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004562b7bd. 00000100:00000001:1.0:1784948752.821084:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305692203 00000100:00000001:3.0:1784948752.821096:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1784948752.821098:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000c54be64. 00000020:00000010:3.0:1784948752.821107:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004562b7bd. 00000100:00000001:1.0:1784948752.821111:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:3.0:1784948752.821118:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000100:00000001:1.0:1784948752.821124:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305692203 00000020:00000010:3.0:1784948752.821128:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013785842. 00000100:00000010:1.0:1784948752.821139:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000092889e7d. 00000020:00000010:3.0:1784948752.821142:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:1.0:1784948752.821151:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000040:3.0:1784948752.821158:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948752.821165:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948752.821165:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948752.821173:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948752.821174:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948752.821180:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948752.821184:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948752.821193:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948752.821196:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948752.821210:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948752.821211:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948752.821217:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948752.821225:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948752.821232:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948752.821235:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948752.821237:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948752.821240:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948752.821249:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948752.821259:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 696 00000100:00000040:3.0:1784948752.821267:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948752.821277:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180528208 : -115539529023408 : ffff96eadb6a4050) 00000100:00000040:3.0:1784948752.821289:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001e461ac8 x1871651289650304/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:413/0 lens 224/0 e 0 to 0 dl 1784948763 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948752.821301:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948752.821305:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004562b7bd. 00000100:00000001:3.0:1784948752.821308:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948752.821313:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004562b7bd. 00000100:00100000:3.0:1784948752.821325:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001e461ac8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289650304:12345-0@lo:400: 00000100:00000200:3.0:1784948752.821330:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650304 00000020:00000001:3.0:1784948752.821334:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000200:0.0:1784948752.821337:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:3.0:1784948752.821340:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948752.821346:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948752.821349:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009bb74647. 00000020:00000001:3.0:1784948752.821351:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948752.821355:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784948752.821361:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:0.0:1784948752.821364:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784948752.821365:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948752.821372:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948752.821378:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948752.821384:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948752.821386:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784948752.821389:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:1.0:1784948752.821395:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948752.821396:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:0.0:1784948752.821398:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 02000000:00000001:3.0:1784948752.821403:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000010:0.0:1784948752.821406:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 02000000:00000010:3.0:1784948752.821411:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000005b888794. 00000800:00000001:1.0:1784948752.821414:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948752.821418:0:337112:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784948752.821419:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948752.821421:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948752.821427:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948752.821431:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948752.821434:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948752.821435:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948752.821444:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650304 00000400:00000200:1.0:1784948752.821452:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:3.0:1784948752.821454:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948752.821468:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001e461ac8 x1871651289650304/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:413/0 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948752.821479:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435500 00000100:00000001:0.0:1784948752.821486:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948752.821500:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 151568 00000100:00000040:0.0:1784948752.821502:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948752.821505:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948752.821511:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948752.821515:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:3.0:1784948752.821521:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=74 v=5 (1 1 1 1) 00000800:00000001:1.0:1784948752.821528:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948752.821534:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784948752.821537:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784948752.821543:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948752.821545:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948752.821548:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:1.0:1784948752.821554:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784948752.821557:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784948752.821558:0:337112:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1784948752.821562:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948752.821563:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948752.821570:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784948752.821571:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:1.0:1784948752.821579:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a8233e67. 00000100:00000040:3.0:1784948752.821582:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:1.0:1784948752.821588:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000a8233e67 x1871651289650432 msgsize 224 00000400:00000010:3.0:1784948752.821592:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000100:00000200:3.0:1784948752.821602:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289650304, offset 224 00000100:00100000:1.0:1784948752.821608:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:3.0:1784948752.821616:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948752.821627:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948752.821631:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:1.0:1784948752.821640:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948752.821647:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435480 00000800:00000001:1.0:1784948752.821648:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948752.821661:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x81f5 [1] + 224 00000400:00000200:3.0:1784948752.821671:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948752.821678:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948752.821683:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948752.821685:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784948752.821694:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650432 00000100:00000200:3.0:1784948752.821695:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948752.821704:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948752.821712:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784948752.821718:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948752.821720:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948752.821733:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948752.821741:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650432 00000020:00000001:0.0:1784948752.821745:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948752.821749:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:0.0:1784948752.821750:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948752.821758:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784948752.821761:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000020:00000040:0.0:1784948752.821763:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000010:3.0:1784948752.821767:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000020:00000001:0.0:1784948752.821777:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948752.821781:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948752.821782:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784948752.821786:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948752.821788:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948752.821790:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:1.0:1784948752.821797:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784948752.821800:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000001:1.0:1784948752.821803:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784948752.821805:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948752.821809:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002eadc14f. 00000020:00000010:0.0:1784948752.821813:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000043fe70d5. 00000400:00000200:1.0:1784948752.821817:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007d821c01 00000100:00000040:3.0:1784948752.821819:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948752.821823:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948752.821824:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007d821c01. 00010000:00000001:3.0:1784948752.821831:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784948752.821833:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784948752.821836:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000043fe70d5. 00000020:00000001:3.0:1784948752.821839:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948752.821843:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948752.821849:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000010:0.0:1784948752.821851:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000020:00000010:0.0:1784948752.821865:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a3c05ade. 00000100:00000040:3.0:1784948752.821865:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001e461ac8 x1871651289650304/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:413/0 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948752.821869:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948752.821880:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948752.821898:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948752.821942:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948752.821945:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948752.821953:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784948752.821959:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784948752.821960:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948752.821970:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948752.821971:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:3.0:1784948752.821971:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001e461ac8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289650304:12345-0@lo:400: Request processed in 660us (1276us total) trans 0 rc 0/0 00000100:00000001:1.0:1784948752.821977:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948752.821980:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948752.821986:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948752.821994:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00001000:1.0:1784948752.822000:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=74 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948752.822025:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=74 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948752.822029:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948752.822029:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948752.822034:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948752.822037:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 696 00000100:00000001:0.0:1784948752.822038:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948752.822040:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784948752.822045:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948752.822050:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948752.822051:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948752.822061:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000040:1.0:1784948752.822066:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948752.822067:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784948752.822069:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948752.822076:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013785842. 00000100:00000001:1.0:1784948752.822090:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948752.822090:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000100:00000001:0.0:1784948752.822096:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948752.822102:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:3.0:1784948752.822103:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000100:00000001:0.0:1784948752.822104:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948752.822106:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784948752.822117:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000040:1.0:1784948752.822119:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948752.822130:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 672 02000000:00000001:3.0:1784948752.822131:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948752.822138:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000005b888794. 00000100:00000040:0.0:1784948752.822146:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:3.0:1784948752.822148:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948752.822156:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948752.822159:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000097a95a31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289650304:0@lo:400: 00000100:00000001:0.0:1784948752.822161:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451649664 : -115544257901952 : ffff96e9c18d4a80) 00000100:00000200:3.0:1784948752.822164:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:3.0:1784948752.822177:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a23805e4. 00000100:00000001:1.0:1784948752.822181:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948752.822186:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a8233e67 x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:413/0 lens 224/0 e 0 to 0 dl 1784948763 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948752.822187:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1784948752.822188:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e9d273c9. 00000100:00000040:1.0:1784948752.822199:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000097a95a31 x1871651289650304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948752.822241:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784948752.822242:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:1.0:1784948752.822245:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948752.822249:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006620b0a1. 00000020:00000010:0.0:1784948752.822252:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000043fe70d5. 02000000:00000001:1.0:1784948752.822254:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948752.822259:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948752.822265:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948752.822266:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948752.822270:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948752.822275:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000eafb2cdc. 00000020:00000010:0.0:1784948752.822278:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000043fe70d5. 02000000:00000001:1.0:1784948752.822282:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948752.822288:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948752.822290:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000097a95a31. 00000100:00000001:1.0:1784948752.822298:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00100000:0.0:1784948752.822304:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a8233e67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289650432:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948752.822304:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948752.822311:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948752.822320:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:0.0:1784948752.822322:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650432 00000100:00000001:1.0:1784948752.822325:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948752.822332:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784948752.822332:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948752.822338:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948752.822342:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948752.822349:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948752.822355:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948752.822358:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948752.822364:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948752.822368:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948752.822371:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948752.822373:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948752.822376:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948752.822380:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948752.822384:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948752.822387:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948752.822392:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000012192387. 02000000:00000001:0.0:1784948752.822397:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948752.822400:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948752.822403:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948752.822406:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650432 00010000:00000001:0.0:1784948752.822410:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948752.822417:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a8233e67 x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:413/0 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948752.822430:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948752.822434:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948752.822437:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=69 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948752.822442:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948752.822446:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948752.822450:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948752.822454:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948752.822457:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948752.822461:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948752.822465:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948752.822470:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000100:00000200:0.0:1784948752.822475:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289650432, offset 224 00000400:00000200:0.0:1784948752.822483:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948752.822496:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948752.822504:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3199:3199:239:4294967295] 192.168.202.137@tcp LPNI seq info [2075:2075:8:4294967295] 00000400:00000200:0.0:1784948752.822519:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948752.822528:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948752.822534:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b51ad697. 00000800:00000200:0.0:1784948752.822542:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948752.822550:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948752.822555:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b51ad697 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948752.822576:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948752.822581:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948752.822585:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948752.822588:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948752.822591:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948752.822597:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a8233e67 x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:413/0 lens 224/224 e 0 to 0 dl 1784948763 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948752.822612:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a8233e67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289650432:12345-192.168.202.137@tcp:400: Request processed in 370us (1022us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948752.822626:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 672 00000100:00000040:0.0:1784948752.822630:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948752.822634:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948752.822636:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948752.822641:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a3c05ade. 00000020:00000010:0.0:1784948752.822647:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948752.822654:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000020:00000040:0.0:1784948752.822659:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948752.822664:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948752.822779:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948752.822786:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b51ad697. 00000400:00000200:3.0:1784948752.822791:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948752.822799:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948752.822805:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000400:00000010:3.0:1784948752.822807:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000001:3.0:1784948752.822812:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948752.822814:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948752.822819:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948752.822834:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948752.822840:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948752.822844:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948752.822854:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948752.822869:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435500 00000400:00000200:0.0:1784948752.822878:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x81fd [1] + 224 00000800:00000001:0.0:1784948752.822886:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948752.822897:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948752.822901:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948752.822938:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948752.822946:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948752.822951:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948752.822968:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948752.822994:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948752.823030:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948752.823036:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948752.823201:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948752.823206:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948752.823214:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=69 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948752.823221:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=69 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948752.823226:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948752.823228:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948752.823232:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948752.823239:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948752.823254:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948752.823259:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948752.823262:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948752.823266:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948752.823269:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948752.823275:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948752.823289:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948752.823294:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948752.823296:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948752.823303:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948752.823318:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a191ee5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289650432:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948752.823327:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948752.823331:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948752.823336:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a191ee5f x1871651289650432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948752.823349:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948752.823353:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948752.823356:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000f1904ca. 02000000:00000001:2.0:1784948752.823362:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948752.823365:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948752.823368:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948752.823372:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948752.823376:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000086e4d21. 02000000:00000001:2.0:1784948752.823381:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948752.823384:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948752.823386:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a191ee5f. 00000100:00000001:2.0:1784948752.823392:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948752.823394:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948752.823398:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948752.823405:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948752.823407:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948752.823412:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948752.823415:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948752.864050:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305692203, at 4305692204 00000100:00000010:1.0:1784948752.864071:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000040:2.0:1784948752.864088:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305692203, at 4305692204 00000100:00000040:3.0:1784948752.864090:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305692203, at 4305692204 00000100:00000040:0.0:1784948752.864103:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305692203, at 4305692204 00000100:00000010:3.0:1784948752.864109:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000092889e7d. 00000100:00000010:2.0:1784948752.864114:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000c54be64. 00000100:00000010:0.0:1784948752.864126:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00010000:00000001:2.0:1784948753.652012:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948753.652019:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948753.652025:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948753.652026:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948753.652101:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948753.652105:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948753.652108:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948753.843989:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948753.844015:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948753.844017:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948753.844021:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948753.844024:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948753.844027:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948753.844029:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948754.676120:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948754.676133:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948754.676145:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948754.676148:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948754.676219:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948754.676226:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948754.676231:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948754.868032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948754.868051:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948754.868053:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948754.868058:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948754.868061:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948754.868064:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948754.868067:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1784948755.380030:0:343910:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.13:988 00000400:00000100:3.0:1784948755.380030:0:343753:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.15:988 00000400:00000100:0.0:1784948755.380033:0:343911:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.24:988 00000400:00000100:1.0:1784948755.380142:0:343910:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.13@tcp at host 111.222.142.13:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:0.0:1784948755.380144:0:343911:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.24@tcp at host 111.222.142.24:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784948755.380144:0:343753:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.15@tcp at host 111.222.142.15:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948755.380157:0:343910:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.13@tcp: down 00000400:00000200:0.0:1784948755.380165:0:343911:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.24@tcp: down 00000400:00000200:3.0:1784948755.380166:0:343753:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.15@tcp: down 00000400:00000200:1.0:1784948755.380168:0:343910:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784948755.380179:0:343910:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000200:0.0:1784948755.380180:0:343911:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.142.24@tcp down 00000400:00000200:3.0:1784948755.380183:0:343753:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:1.0:1784948755.380186:0:343910:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006c136176. 00000800:00000100:3.0:1784948755.380193:0:343753:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000800:00000010:3.0:1784948755.380196:0:343753:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:1.0:1784948755.380197:0:343910:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948755.380203:0:343753:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948755.380207:0:343910:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948755.380213:0:343753:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948755.380217:0:343910:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11028 health :900 00000400:00000200:1.0:1784948755.380223:0:343910:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948755.380227:0:343910:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:3.0:1784948755.380228:0:343753:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948755.380233:0:343910:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:3.0:1784948755.380234:0:343753:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:3.0:1784948755.380250:0:343753:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:2.0:1784948755.380282:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948755.380290:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:2.0:1784948755.380296:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948755.380298:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948755.380300:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000400:00000010:2.0:1784948755.380303:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000400:00000200:2.0:1784948755.380307:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948755.380310:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:2.0:1784948755.380314:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000059e5b358. 00000400:00000200:2.0:1784948755.380318:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:2.0:1784948755.380320:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:2.0:1784948755.380323:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:2.0:1784948755.380326:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:2.0:1784948755.380328:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948755.380329:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948755.380331:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:2.0:1784948755.380332:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000400:00000200:2.0:1784948755.380334:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948755.380336:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:2.0:1784948755.380337:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004041a48e. 00000400:00000200:2.0:1784948755.380341:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:2.0:1784948755.380343:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:2.0:1784948755.380345:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784948755.700047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948755.700057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948755.700065:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948755.700066:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948755.700165:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948755.700174:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948755.700178:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948755.892163:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948755.892187:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948755.892190:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948755.892198:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948755.892202:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948755.892207:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948755.892210:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948756.084111:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784948756.084127:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a666de65. 00000400:00000010:3.0:1784948756.084142:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000208850ab. 00000400:00000010:3.0:1784948756.084152:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000400:00000010:3.0:1784948756.084165:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007b677d5c. 00000400:00000200:3.0:1784948756.084175:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948756.084198:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948756.084209:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948756.084216:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3200:3200:241:4294967295] 192.168.202.137@tcp LPNI seq info [2076:2076:8:4294967295] 00000400:00000200:3.0:1784948756.084228:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948756.084247:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948756.084255:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948756.084262:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948756.084271:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948756.084281:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948756.084286:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784948756.084313:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11030 health :800 00000800:00000200:1.0:1784948756.084494:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948756.084503:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:1.0:1784948756.084515:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948756.084527:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784948756.084537:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:3.0:1784948756.084545:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948756.084546:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948756.084570:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948756.084577:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948756.084587:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948756.084602:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948756.084620:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784948756.084632:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784948756.084645:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948756.084654:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3201:3201:241:4294967295] 192.168.202.137@tcp LPNI seq info [2077:2077:8:4294967295] 00000400:00000200:3.0:1784948756.084671:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784948756.084680:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948756.084688:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948756.084696:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948756.084704:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948756.084709:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948756.084801:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784948756.084877:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948756.084895:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:3.0:1784948756.084901:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948756.084950:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948756.084953:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948756.084967:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784948756.084969:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948756.084998:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948756.085020:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948756.085031:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784948756.085050:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8211 00000800:00000001:1.0:1784948756.085058:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948756.085067:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948756.085069:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948756.085074:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784948756.085080:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:1.0:1784948756.085084:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000400:00000200:1.0:1784948756.085087:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784948756.085091:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a666de65. 00000400:00000010:1.0:1784948756.085095:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000208850ab. 00000800:00000001:1.0:1784948756.085103:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948756.085106:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948756.723989:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948756.723996:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948756.724002:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948756.724003:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948756.724067:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948756.724072:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948756.724075:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948756.916022:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948756.916036:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948756.916038:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948756.916044:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948756.916047:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948756.916049:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948756.916051:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948757.748087:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948757.748096:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948757.748103:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948757.748105:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948757.748173:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948757.748178:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948757.748181:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784948757.940055:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948757.940066:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948757.940075:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005e465757. 00000020:00000040:0.0:1784948757.940082:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948757.940087:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948757.940088:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948757.940094:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948757.940100:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948757.940100:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948757.940106:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784948757.940111:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d1b12746. 00000100:00000001:1.0:1784948757.940111:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948757.940114:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948757.940119:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948757.940121:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948757.940122:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948757.940136:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948757.940148:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948757.940151:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948757.940156:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e465757] to pc [ptlrpcd_00_03+3] req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948757.940188:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948757.940191:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948757.940194:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948757.940198:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948757.940202:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948757.940205:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000075b1f5f6. 00000020:00000040:0.0:1784948757.940208:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948757.940211:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948757.940212:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948757.940214:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948757.940216:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948757.940218:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:0.0:1784948757.940223:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b84187a1. 00000100:00000001:1.0:1784948757.940223:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948757.940243:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948757.940247:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948757.940249:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948757.940251:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948757.940260:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948757.940294:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948757.940297:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948757.940300:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948757.940301:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948757.940310:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948757.940312:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b1f5f6] to pc [ptlrpcd_00_00+0] req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948757.940316:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948757.940325:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948757.940336:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005e465757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289650560:0@lo:400: 00000100:00000001:1.0:1784948757.940341:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948757.940347:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948757.940350:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948757.940353:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948757.940357:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948757.940369:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 02000000:00000001:1.0:1784948757.940374:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948757.940377:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11034) 00000400:00000010:1.0:1784948757.940382:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eadc14f. 00000100:00000001:3.0:1784948757.940385:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948757.940391:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:1.0:1784948757.940394:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007d821c01. 00000100:00000001:3.0:1784948757.940394:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948757.940407:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948757.940408:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289650560, portal 25 00000100:00000001:1.0:1784948757.940414:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948757.940420:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948757.940420:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558305664 : -115544151245952 : ffff96e9c7e8bb80) 00000100:00000001:0.0:1784948757.940429:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948757.940432:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784948757.940436:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948757.940438:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948757.940447:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948757.940453:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948757.940455:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784948757.940459:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948757.940464:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00100000:3.0:1784948757.940485:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000075b1f5f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289650688:192.168.202.137@tcp:400: 00000400:00000010:1.0:1784948757.940487:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784948757.940494:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289650560, offset 0 00000100:00000001:3.0:1784948757.940495:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948757.940501:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948757.940504:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948757.940506:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784948757.940508:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:3.0:1784948757.940512:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 02000000:00000001:3.0:1784948757.940516:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948757.940522:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e686bcb0. 00000400:00000200:1.0:1784948757.940526:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:3.0:1784948757.940535:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000400:00000200:1.0:1784948757.940542:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435580 00000100:00000200:3.0:1784948757.940544:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289650688, portal 10 00000100:00000001:3.0:1784948757.940550:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784948757.940554:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x77f5 [2] + 0 00000100:00000001:3.0:1784948757.940556:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558301184 : -115544151250432 : ffff96e9c7e8aa00) 00000400:00000010:1.0:1784948757.940561:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b32476f7. 00000400:00000200:1.0:1784948757.940568:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000099bab33a 00000100:00000040:3.0:1784948757.940573:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948757.940577:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948757.940587:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948757.940597:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000099bab33a 00000100:00000001:3.0:1784948757.940600:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784948757.940601:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000099bab33a. 00000100:00000040:3.0:1784948757.940608:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948757.940609:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948757.940613:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:3.0:1784948757.940614:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000100:00000040:1.0:1784948757.940619:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000322d6a4a x1871651289650560 msgsize 224 00000100:00000200:3.0:1784948757.940621:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289650688, offset 0 00000100:00100000:1.0:1784948757.940628:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784948757.940631:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948757.940635:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948757.940646:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784948757.940652:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948757.940652:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948757.940660:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948757.940664:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948757.940668:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948757.940673:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3202:3202:241:4294967295] 192.168.202.137@tcp LPNI seq info [2078:2078:8:4294967295] 00000100:00000001:2.0:1784948757.940694:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948757.940699:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650560 02000000:00000001:2.0:1784948757.940703:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948757.940708:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948757.940713:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948757.940713:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:2.0:1784948757.940719:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948757.940725:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00100000:2.0:1784948757.940726:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650560 00000400:00000200:1.0:1784948757.940730:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000020:00000001:2.0:1784948757.940731:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948757.940734:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000010:3.0:1784948757.940737:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000400:00000010:1.0:1784948757.940738:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000020:00000001:2.0:1784948757.940738:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948757.940744:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948757.940748:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1784948757.940749:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:3.0:1784948757.940754:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:2.0:1784948757.940757:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000200:1.0:1784948757.940758:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948757.940766:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:3.0:1784948757.940770:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000010:2.0:1784948757.940774:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000046d06efb. 00000800:00000200:3.0:1784948757.940779:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948757.940792:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948757.940795:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1784948757.940798:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000046d06efb. 00000100:00000040:1.0:1784948757.940801:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948757.940803:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948757.940807:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f1904ca. 00000100:00000040:3.0:1784948757.940813:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2038 usecs (3 jiffies) 00000020:00000010:2.0:1784948757.940817:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bee21534. 00000100:00000001:3.0:1784948757.940821:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305697283 00000100:00000001:1.0:1784948757.940826:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948757.940826:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000010:3.0:1784948757.940829:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:1.0:1784948757.940834:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948757.940839:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000040:2.0:1784948757.940841:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948757.940844:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948757.940848:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948757.940851:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948757.940852:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305697283 00000100:00000001:1.0:1784948757.940853:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305697322 00000100:00000001:2.0:1784948757.940857:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948757.940859:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:3.0:1784948757.940866:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948757.940869:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948757.940872:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305697283 00000100:00000010:3.0:1784948757.940877:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:3.0:1784948757.940882:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948757.940883:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948757.940886:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305697283 00000100:00000010:3.0:1784948757.940892:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:3.0:1784948757.940898:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948757.940900:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948757.940935:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948757.940937:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784948757.940943:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 697 00000100:00000001:3.0:1784948757.940945:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948757.940951:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948757.940952:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948757.940958:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180541520 : -115539529010096 : ffff96eadb6a7450) 00000100:00000001:3.0:1784948757.940958:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948757.940990:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1784948757.940992:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000322d6a4a x1871651289650560/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:418/0 lens 224/0 e 0 to 0 dl 1784948768 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948757.940993:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948757.941001:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948757.941005:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948757.941010:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948757.941010:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948757.941017:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948757.941018:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000046d06efb. 00000100:00000001:2.0:1784948757.941023:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948757.941026:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000046d06efb. 00000100:00100000:2.0:1784948757.941033:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000322d6a4a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289650560:12345-0@lo:400: 00000100:00000200:2.0:1784948757.941037:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650560 00000020:00000001:2.0:1784948757.941040:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948757.941055:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948757.941058:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948757.941061:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948757.941063:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948757.941066:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948757.941069:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948757.941071:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948757.941074:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948757.941077:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948757.941080:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948757.941085:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948757.941089:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948757.941095:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000e96c6c0. 02000000:00000001:2.0:1784948757.941100:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948757.941105:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948757.941108:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948757.941113:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650560 00000800:00000200:0.0:1784948757.941117:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784948757.941118:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:3.0:1784948757.941120:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000200:2.0:1784948757.941127:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000322d6a4a x1871651289650560/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:418/0 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1784948757.941128:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000800:00000001:3.0:1784948757.941135:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948757.941140:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948757.941142:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948757.941146:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305697283 00000800:00000001:3.0:1784948757.941151:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948757.941156:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305697323 00000400:00000200:0.0:1784948757.941157:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000001:2.0:1784948757.941168:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784948757.941169:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305697283 00000400:00000200:3.0:1784948757.941169:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948757.941170:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00010000:00000001:2.0:1784948757.941176:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948757.941177:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000100:00000001:1.0:1784948757.941179:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305697323 00000100:00000001:0.0:1784948757.941188:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00001000:2.0:1784948757.941188:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=79 v=5 (1 1 1 1) 00000400:00000200:3.0:1784948757.941189:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435600 00000100:00000200:0.0:1784948757.941199:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948757.941219:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1784948757.941226:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 151792 00000100:00000040:2.0:1784948757.941230:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948757.941240:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:3.0:1784948757.941245:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784948757.941248:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784948757.941252:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1784948757.941253:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948757.941259:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948757.941260:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948757.941262:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948757.941267:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784948757.941271:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948757.941280:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000200:3.0:1784948757.941285:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:2.0:1784948757.941290:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289650560, offset 224 00000100:00000001:0.0:1784948757.941297:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948757.941317:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948757.941318:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:2.0:1784948757.941323:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000200:3.0:1784948757.941323:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:3.0:1784948757.941331:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000adacbaf0. 00000100:00000040:3.0:1784948757.941334:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000adacbaf0 x1871651289650688 msgsize 224 00000400:00000200:2.0:1784948757.941340:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:3.0:1784948757.941349:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948757.941356:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305697283 00000400:00000200:2.0:1784948757.941358:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435580 00000100:00000001:3.0:1784948757.941366:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948757.941368:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305697323 00000400:00000200:2.0:1784948757.941373:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8215 [1] + 224 00000100:00000001:0.0:1784948757.941376:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305697283 00000400:00000200:2.0:1784948757.941381:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948757.941383:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948757.941389:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948757.941390:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:3.0:1784948757.941390:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948757.941391:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:3.0:1784948757.941398:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948757.941399:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650688 00000100:00000001:0.0:1784948757.941400:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948757.941402:0:342757:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1784948757.941405:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948757.941409:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948757.941409:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784948757.941413:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948757.941415:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948757.941419:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948757.941425:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 02000000:00000001:1.0:1784948757.941425:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948757.941432:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948757.941435:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948757.941435:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650688 00000100:00000001:0.0:1784948757.941440:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1784948757.941442:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948757.941446:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948757.941452:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948757.941454:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1784948757.941457:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948757.941461:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948757.941464:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948757.941466:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948757.941470:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784948757.941473:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948757.941475:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000400:00000200:2.0:1784948757.941478:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000020:00000010:1.0:1784948757.941482:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000015713dee. 00000400:00000010:2.0:1784948757.941483:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000100:00000001:2.0:1784948757.941498:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000010:1.0:1784948757.941499:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004db98700. 00000400:00000001:0.0:1784948757.941500:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784948757.941501:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:0.0:1784948757.941507:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:1.0:1784948757.941508:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948757.941510:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948757.941512:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002eadc14f. 00000100:00000040:2.0:1784948757.941518:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:0.0:1784948757.941520:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007d821c01 00000100:00000040:1.0:1784948757.941522:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00010000:00000001:2.0:1784948757.941523:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:0.0:1784948757.941526:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007d821c01. 00000020:00000001:2.0:1784948757.941528:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948757.941529:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948757.941531:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784948757.941533:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:2.0:1784948757.941535:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784948757.941538:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948757.941540:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948757.941544:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000322d6a4a x1871651289650560/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:418/0 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948757.941546:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948757.941550:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948757.941553:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948757.941574:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948757.941585:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948757.941589:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000322d6a4a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289650560:12345-0@lo:400: Request processed in 573us (964us total) trans 0 rc 0/0 00000100:00000001:1.0:1784948757.941590:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948757.941596:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948757.941608:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 673 00000100:00100000:2.0:1784948757.941614:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 697 00000100:00000040:1.0:1784948757.941620:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948757.941621:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1784948757.941621:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948757.941627:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948757.941628:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482528768 : -115544227022848 : ffff96e9c3647800) 00000100:00000001:2.0:1784948757.941629:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948757.941634:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:2.0:1784948757.941635:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:0.0:1784948757.941639:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000040:1.0:1784948757.941640:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000adacbaf0 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:418/0 lens 224/0 e 0 to 0 dl 1784948768 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948757.941643:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:2.0:1784948757.941643:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bee21534. 00000100:00000001:0.0:1784948757.941649:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948757.941652:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 02000000:00000001:0.0:1784948757.941656:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948757.941660:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f1904ca. 00000100:00001000:0.0:1784948757.941665:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=79 v=5 (5 5 5 5) 00000020:00000040:2.0:1784948757.941669:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784948757.941670:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948757.941677:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784948757.941680:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00001000:0.0:1784948757.941682:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=79 v=5 (1 1 1 1) 00000020:00000010:1.0:1784948757.941684:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00000001:0.0:1784948757.941690:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948757.941693:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948757.941694:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:0.0:1784948757.941696:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000010:2.0:1784948757.941699:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000000e96c6c0. 00010000:00000001:0.0:1784948757.941701:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948757.941702:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000adacbaf0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289650688:12345-192.168.202.137@tcp:400: 02000000:00000001:2.0:1784948757.941706:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948757.941708:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948757.941710:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948757.941718:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948757.941720:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000200:1.0:1784948757.941721:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650688 00000400:00000010:2.0:1784948757.941729:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000aee1f7f7. 00000020:00000001:1.0:1784948757.941731:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:2.0:1784948757.941738:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000553d6b82. 00000020:00000001:1.0:1784948757.941739:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784948757.941740:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948757.941746:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948757.941747:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948757.941750:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948757.941755:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:0.0:1784948757.941757:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948757.941760:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948757.941771:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00100000:0.0:1784948757.941773:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005e465757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289650560:0@lo:400: 00000020:00000001:1.0:1784948757.941777:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948757.941779:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948757.941783:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784948757.941784:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784948757.941787:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:0.0:1784948757.941790:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005e465757 x1871651289650560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948757.941792:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948757.941798:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948757.941803:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948757.941805:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948757.941807:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:0.0:1784948757.941809:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948757.941813:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000010:1.0:1784948757.941813:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000101d5fea. 02000000:00000001:0.0:1784948757.941817:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1784948757.941819:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948757.941821:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1784948757.941823:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948757.941824:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784948757.941826:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948757.941830:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000040:1.0:1784948757.941831:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650688 02000000:00000010:0.0:1784948757.941833:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d1b12746. 00010000:00000001:1.0:1784948757.941835:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:0.0:1784948757.941839:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948757.941843:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00010000:00000200:1.0:1784948757.941845:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000adacbaf0 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:418/0 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784948757.941846:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005e465757. 00000100:00000001:0.0:1784948757.941853:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948757.941855:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948757.941861:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948757.941865:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948757.941867:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1784948757.941869:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948757.941870:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948757.941874:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=74 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948757.941875:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948757.941879:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948757.941882:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948757.941886:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948757.941889:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948757.941892:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948757.941894:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948757.941896:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948757.941899:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948757.941928:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784948757.941933:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289650688, offset 224 00000400:00000200:1.0:1784948757.941937:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948757.941944:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948757.941949:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3203:3203:241:4294967295] 192.168.202.137@tcp LPNI seq info [2079:2079:8:4294967295] 00000400:00000200:1.0:1784948757.941957:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948757.941983:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948757.942014:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008f5595bf. 00000800:00000200:1.0:1784948757.942018:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948757.942024:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948757.942027:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f5595bf type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948757.942042:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948757.942045:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948757.942048:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948757.942050:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948757.942052:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948757.942056:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000adacbaf0 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:418/0 lens 224/224 e 0 to 0 dl 1784948768 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948757.942066:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000adacbaf0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289650688:12345-192.168.202.137@tcp:400: Request processed in 394us (729us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948757.942075:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 673 00000100:00000040:1.0:1784948757.942079:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948757.942081:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948757.942084:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948757.942087:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004db98700. 00000020:00000010:1.0:1784948757.942091:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948757.942095:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000015713dee. 00000020:00000040:1.0:1784948757.942100:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948757.942103:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948757.942165:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948757.942170:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008f5595bf. 00000400:00000200:2.0:1784948757.942174:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948757.942181:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948757.942186:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000800:00000001:0.0:1784948757.942189:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784948757.942189:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:2.0:1784948757.942197:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948757.942198:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948757.942202:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948757.942207:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948757.942210:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948757.942218:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948757.942245:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435600 00000400:00000200:0.0:1784948757.942250:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x821d [1] + 224 00000800:00000001:0.0:1784948757.942255:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948757.942261:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948757.942262:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948757.942266:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948757.942270:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948757.942273:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948757.942281:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948757.942297:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948757.942302:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948757.942304:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948757.942355:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948757.942368:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948757.942373:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948757.942374:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948757.942376:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e686bcb0. 00000400:00000200:3.0:1784948757.942381:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:3.0:1784948757.942383:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:3.0:1784948757.942385:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948757.942390:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948757.942399:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948757.942406:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948757.942407:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948757.942409:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948757.942410:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948757.942412:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948757.942413:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948757.942415:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948757.942419:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=74 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948757.942422:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=74 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948757.942424:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948757.942435:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948757.942436:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948757.942439:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948757.942446:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948757.942448:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948757.942449:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948757.942451:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948757.942452:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948757.942454:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948757.942466:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948757.942468:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948757.942469:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948757.942472:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948757.942478:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000075b1f5f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289650688:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948757.942481:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948757.942483:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948757.942484:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000075b1f5f6 x1871651289650688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948773 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948757.942490:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948757.942491:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948757.942492:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:3.0:1784948757.942495:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948757.942496:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948757.942498:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948757.942500:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948757.942501:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b84187a1. 02000000:00000001:3.0:1784948757.942503:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948757.942504:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948757.942505:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000075b1f5f6. 00000100:00000001:3.0:1784948757.942508:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948757.942509:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948757.942511:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948757.942514:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948757.942515:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948757.942517:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948757.942518:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948757.984046:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305697323, at 4305697324 00000100:00000040:2.0:1784948757.984070:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305697323, at 4305697324 00000100:00000010:3.0:1784948757.984071:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00000100:00000040:0.0:1784948757.984080:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305697322, at 4305697324 00000100:00000010:2.0:1784948757.984087:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000040:1.0:1784948757.984088:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305697323, at 4305697324 00000100:00000010:0.0:1784948757.984102:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:1.0:1784948757.984106:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00010000:00000001:2.0:1784948758.771985:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948758.771992:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948758.771998:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948758.771999:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948758.772035:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948758.772040:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948758.772043:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948758.964052:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948758.964067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948758.964069:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948758.964075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948758.964078:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948758.964082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948758.964085:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948759.795993:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948759.796003:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948759.796011:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948759.796015:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948759.796017:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948759.796021:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948759.796025:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948759.796027:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948759.796030:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948759.796032:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948759.796083:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948759.796088:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948759.796092:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948759.987999:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948759.988013:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948759.988015:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948759.988019:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948759.988022:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948759.988026:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948759.988028:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948760.180039:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:0.0:1784948760.180051:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000800:00000200:0.0:1784948760.180058:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000800:00000200:3.0:1784948760.180239:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784948760.820048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948760.820058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948760.820068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948760.820070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948760.820151:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948760.820156:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948760.820160:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948761.012123:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948761.012140:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948761.012142:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948761.012148:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948761.012151:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948761.012155:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948761.012158:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948761.844043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948761.844053:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948761.844061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948761.844063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948761.844156:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948761.844162:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948761.844166:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948762.036039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948762.036056:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948762.036059:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948762.036064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948762.036067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948762.036071:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948762.036075:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948762.868035:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948762.868043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948762.868049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948762.868051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948762.868120:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948762.868126:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948762.868129:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948763.060031:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948763.060043:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948763.060045:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948763.060049:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948763.060052:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948763.060055:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948763.060057:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948763.060069:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948763.060075:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948763.060081:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000049907b7f. 00000020:00000040:2.0:1784948763.060087:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948763.060091:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948763.060093:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948763.060098:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948763.060104:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000263f8d59. 00000100:00000001:2.0:1784948763.060110:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948763.060118:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948763.060127:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948763.060129:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948763.060134:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049907b7f] to pc [ptlrpcd_00_01+1] req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948763.060178:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.060181:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1784948763.060181:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:0.0:1784948763.060186:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1784948763.060187:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1784948763.060191:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948763.060193:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948763.060197:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000040:0.0:1784948763.060199:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784948763.060201:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005db4bd4e. 00000100:00000001:3.0:1784948763.060202:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1784948763.060208:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948763.060208:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948763.060214:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948763.060215:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948763.060218:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948763.060218:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948763.060221:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948763.060224:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:2.0:1784948763.060226:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000016632059. 02000000:00000001:0.0:1784948763.060227:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948763.060230:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948763.060231:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948763.060237:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000049907b7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289650816:0@lo:400: 00000100:00000040:2.0:1784948763.060239:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948763.060243:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948763.060249:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784948763.060250:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1784948763.060252:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948763.060253:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948763.060256:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784948763.060259:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005db4bd4e] to pc [ptlrpcd_00_02+2] req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948763.060261:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eff166f6. 02000000:00000001:0.0:1784948763.060266:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948763.060274:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001c023af. 00000400:00000010:0.0:1784948763.060282:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000001:2.0:1784948763.060285:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948763.060289:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289650816, portal 25 00000100:00000040:2.0:1784948763.060290:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11040) 00000100:00000001:0.0:1784948763.060295:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948763.060297:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470014336 : -115544239537280 : ffff96e9c2a58380) 00000100:00000001:3.0:1784948763.060298:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948763.060301:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948763.060304:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948763.060305:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948763.060313:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948763.060313:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948763.060315:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948763.060319:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948763.060321:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948763.060327:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000100:00000001:3.0:1784948763.060329:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1784948763.060330:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289650816, offset 0 00000100:00000001:3.0:1784948763.060332:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948763.060336:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948763.060337:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948763.060341:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784948763.060342:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948763.060347:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005db4bd4e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289650944:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948763.060350:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948763.060352:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948763.060353:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948763.060357:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948763.060358:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:3.0:1784948763.060362:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 02000000:00000001:3.0:1784948763.060365:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948763.060369:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e686bcb0. 00000400:00000200:0.0:1784948763.060370:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435680 00000400:00000200:0.0:1784948763.060377:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7819 [2] + 0 00000400:00000010:3.0:1784948763.060377:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000400:00000010:0.0:1784948763.060383:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000035db9be3. 00000400:00000200:0.0:1784948763.060386:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b92e9146 00000400:00000200:0.0:1784948763.060393:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:3.0:1784948763.060395:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289650944, portal 10 00000400:00000200:0.0:1784948763.060399:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948763.060400:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948763.060403:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470023296 : -115544239528320 : ffff96e9c2a5a680) 00000400:00000200:0.0:1784948763.060407:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b92e9146 00000400:00000010:0.0:1784948763.060409:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b92e9146. 00000100:00000040:3.0:1784948763.060413:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948763.060416:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948763.060420:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784948763.060424:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948763.060425:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000030d4261 x1871651289650816 msgsize 224 00000100:00000040:3.0:1784948763.060430:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784948763.060435:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000010:3.0:1784948763.060436:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000100:00000040:0.0:1784948763.060437:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000200:3.0:1784948763.060441:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289650944, offset 0 00000400:00000200:3.0:1784948763.060447:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948763.060451:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948763.060456:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948763.060461:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948763.060465:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948763.060468:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948763.060471:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948763.060475:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3204:3204:241:4294967295] 192.168.202.137@tcp LPNI seq info [2080:2080:8:4294967295] 00000100:00000001:1.0:1784948763.060482:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948763.060487:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650816 00000400:00000200:3.0:1784948763.060487:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948763.060493:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948763.060496:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000800:00000010:3.0:1784948763.060498:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000400:00000010:0.0:1784948763.060499:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000100:00000001:0.0:1784948763.060506:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784948763.060509:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784948763.060512:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948763.060521:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784948763.060524:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784948763.060526:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:0.0:1784948763.060530:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948763.060532:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948763.060538:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948763.060541:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948763.060542:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948763.060548:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2040 usecs (3 jiffies) 02000000:00000001:1.0:1784948763.060552:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948763.060554:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305702403 00000100:00000001:0.0:1784948763.060557:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948763.060560:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650816 00000100:00000010:3.0:1784948763.060561:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:0.0:1784948763.060562:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784948763.060566:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948763.060570:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784948763.060571:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948763.060572:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948763.060575:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948763.060578:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305702442 00000100:00000001:3.0:1784948763.060579:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305702403 00000020:00000040:1.0:1784948763.060580:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948763.060586:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305702403 00000100:00000010:3.0:1784948763.060586:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000afdabce. 00000020:00000001:1.0:1784948763.060587:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948763.060593:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000001:1.0:1784948763.060594:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948763.060596:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305702442 00000100:00000001:3.0:1784948763.060599:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305702403 00000100:00000001:1.0:1784948763.060600:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948763.060601:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305702403 00000100:00000001:0.0:1784948763.060607:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305702442 00000020:00000010:1.0:1784948763.060607:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000010:3.0:1784948763.060608:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:1.0:1784948763.060614:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948763.060615:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000010:1.0:1784948763.060621:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00000001:3.0:1784948763.060622:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305702403 00000100:00000001:0.0:1784948763.060624:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305702403 00000100:00000010:3.0:1784948763.060626:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000010:1.0:1784948763.060628:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000100:00000001:0.0:1784948763.060631:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305702442 00000100:00000001:3.0:1784948763.060633:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000010:1.0:1784948763.060634:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004db98700. 00000020:00000010:1.0:1784948763.060642:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:3.0:1784948763.060642:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948763.060643:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305702403 00000100:00000001:3.0:1784948763.060650:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.060653:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948763.060655:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948763.060655:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.060656:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948763.060660:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948763.060663:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948763.060663:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948763.060664:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948763.060667:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.060670:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948763.060670:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948763.060673:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948763.060675:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784948763.060676:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000100:00000001:3.0:1784948763.060677:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948763.060678:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948763.060683:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:2.0:1784948763.060684:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948763.060685:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948763.060688:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948763.060689:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948763.060692:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948763.060695:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948763.060696:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948763.060697:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948763.060701:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948763.060702:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.060703:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948763.060704:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00000100:00000001:1.0:1784948763.060705:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:2.0:1784948763.060709:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000100:00000001:0.0:1784948763.060712:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948763.060713:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1784948763.060714:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 698 00000100:00000040:1.0:1784948763.060720:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000200:2.0:1784948763.060721:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948763.060724:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180535376 : -115539529016240 : ffff96eadb6a5c50) 00000800:00000001:0.0:1784948763.060733:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784948763.060734:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000030d4261 x1871651289650816/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:424/0 lens 224/0 e 0 to 0 dl 1784948774 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948763.060734:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948763.060741:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948763.060748:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948763.060754:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948763.060759:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:0.0:1784948763.060761:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948763.060761:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948763.060762:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:2.0:1784948763.060764:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948763.060769:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948763.060773:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000400:00000200:0.0:1784948763.060776:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00100000:1.0:1784948763.060781:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000030d4261 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289650816:12345-0@lo:400: 00000100:00000200:1.0:1784948763.060786:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650816 00000400:00000200:0.0:1784948763.060787:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435700 00000020:00000001:1.0:1784948763.060789:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948763.060794:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784948763.060797:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 152016 00000020:00000001:1.0:1784948763.060798:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948763.060804:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:0.0:1784948763.060806:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948763.060806:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948763.060810:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948763.060812:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948763.060813:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948763.060814:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948763.060834:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:0.0:1784948763.060836:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784948763.060840:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948763.060847:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784948763.060848:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948763.060854:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948763.060856:0:337112:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:1.0:1784948763.060859:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000200:0.0:1784948763.060860:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000010:1.0:1784948763.060865:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001cc3fd72. 00000100:00000010:0.0:1784948763.060867:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005e465757. 02000000:00000001:1.0:1784948763.060872:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948763.060873:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000005e465757 x1871651289650944 msgsize 224 00000100:00000001:1.0:1784948763.060878:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948763.060886:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948763.060887:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000040:1.0:1784948763.060893:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650816 00010000:00000001:1.0:1784948763.060897:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948763.060902:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000030d4261 x1871651289650816/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:424/0 lens 224/224 e 0 to 0 dl 1784948774 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948763.060910:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948763.060912:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.060964:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784948763.060972:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948763.060974:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:0.0:1784948763.060975:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784948763.060976:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=85 v=5 (1 1 1 1) 00000100:00100000:2.0:1784948763.060978:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289650944 02000000:00000001:2.0:1784948763.060982:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948763.060985:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948763.060986:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948763.060987:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948763.060992:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 02000000:00000001:2.0:1784948763.060992:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948763.060998:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289650944 00000100:00000001:1.0:1784948763.060999:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:2.0:1784948763.061002:0:343544:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784948763.061004:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000040:2.0:1784948763.061005:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000001:1.0:1784948763.061007:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948763.061008:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784948763.061011:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1784948763.061012:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:1.0:1784948763.061015:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:2.0:1784948763.061017:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948763.061020:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000010:1.0:1784948763.061021:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000100:00000001:2.0:1784948763.061025:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:1.0:1784948763.061026:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289650816, offset 224 00000020:00000010:2.0:1784948763.061029:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006b18ba8f. 00000100:00000001:2.0:1784948763.061033:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948763.061036:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:2.0:1784948763.061037:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006b18ba8f. 00000020:00000010:2.0:1784948763.061044:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f1904ca. 00000400:00000200:1.0:1784948763.061045:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:2.0:1784948763.061049:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000734334de. 00000400:00000200:1.0:1784948763.061055:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435680 00000020:00000010:2.0:1784948763.061055:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000400:00000200:1.0:1784948763.061062:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8231 [1] + 224 00000100:00000040:2.0:1784948763.061064:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948763.061068:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:1.0:1784948763.061069:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948763.061070:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948763.061074:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948763.061076:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948763.061080:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948763.061081:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784948763.061084:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:1.0:1784948763.061087:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948763.061088:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948763.061098:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948763.061100:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948763.061113:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948763.061114:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948763.061118:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784948763.061122:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000100:00100000:2.0:1784948763.061124:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 674 00000400:00000010:1.0:1784948763.061125:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000100:00000040:2.0:1784948763.061127:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948763.061130:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948763.061131:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558305664 : -115544151245952 : ffff96e9c7e8bb80) 00000100:00000001:1.0:1784948763.061134:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000040:2.0:1784948763.061138:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005e465757 x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:424/0 lens 224/0 e 0 to 0 dl 1784948774 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948763.061139:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.061146:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948763.061147:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948763.061149:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948763.061149:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:1.0:1784948763.061152:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948763.061154:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948763.061154:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006b18ba8f. 00000400:00000001:0.0:1784948763.061156:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784948763.061159:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000001:0.0:1784948763.061161:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784948763.061161:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000010:2.0:1784948763.061162:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006b18ba8f. 00000400:00000010:0.0:1784948763.061165:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000001c023af. 00000100:00000040:1.0:1784948763.061168:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000030d4261 x1871651289650816/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:424/0 lens 224/224 e 0 to 0 dl 1784948774 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948763.061172:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000100:00100000:2.0:1784948763.061172:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005e465757 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289650944:12345-192.168.202.137@tcp:400: 00000400:00000010:0.0:1784948763.061177:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:0.0:1784948763.061182:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948763.061183:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289650944 00000020:00000001:2.0:1784948763.061188:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000200:0.0:1784948763.061189:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948763.061191:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00100000:1.0:1784948763.061193:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000030d4261 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289650816:12345-0@lo:400: Request processed in 430us (763us total) trans 0 rc 0/0 00000020:00000001:2.0:1784948763.061195:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948763.061200:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948763.061203:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784948763.061209:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00100000:1.0:1784948763.061213:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 698 00000020:00000001:2.0:1784948763.061213:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948763.061217:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948763.061219:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784948763.061220:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:2.0:1784948763.061222:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948763.061226:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:2.0:1784948763.061227:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948763.061231:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784948763.061232:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1784948763.061232:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948763.061236:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948763.061237:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004db98700. 00000100:00000001:2.0:1784948763.061238:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948763.061239:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948763.061242:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:0.0:1784948763.061244:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:1.0:1784948763.061244:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 02000000:00000010:2.0:1784948763.061246:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000e96c6c0. 00000100:00000001:0.0:1784948763.061247:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:1.0:1784948763.061249:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000100:00000001:0.0:1784948763.061250:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948763.061251:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948763.061255:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:0.0:1784948763.061256:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948763.061256:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948763.061262:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948763.061264:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=85 v=5 (5 5 5 5) 02000000:00000001:1.0:1784948763.061264:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000040:2.0:1784948763.061265:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289650944 02000000:00000010:1.0:1784948763.061268:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000001cc3fd72. 00010000:00000001:2.0:1784948763.061271:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00001000:0.0:1784948763.061273:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=85 v=5 (1 1 1 1) 02000000:00000001:1.0:1784948763.061274:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000200:2.0:1784948763.061278:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005e465757 x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:424/0 lens 224/224 e 0 to 0 dl 1784948774 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948763.061279:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948763.061280:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948763.061283:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948763.061287:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:0.0:1784948763.061289:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948763.061293:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948763.061294:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b32476f7. 00000100:00000001:0.0:1784948763.061298:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948763.061300:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000010:1.0:1784948763.061302:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f0571d9e. 00010000:00000001:2.0:1784948763.061303:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948763.061305:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:2.0:1784948763.061308:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=80 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948763.061317:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948763.061320:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948763.061323:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948763.061325:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948763.061326:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784948763.061327:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948763.061330:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:0.0:1784948763.061333:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948763.061334:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948763.061340:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784948763.061347:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000049907b7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289650816:0@lo:400: 00000100:00000040:2.0:1784948763.061348:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948763.061354:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000100:00000001:0.0:1784948763.061356:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948763.061358:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:2.0:1784948763.061359:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289650944, offset 224 00000100:00000040:0.0:1784948763.061361:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000049907b7f x1871651289650816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948763.061369:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948763.061375:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948763.061378:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:2.0:1784948763.061381:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000010:0.0:1784948763.061382:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eff166f6. 02000000:00000001:0.0:1784948763.061388:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1784948763.061391:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3205:3205:241:4294967295] 192.168.202.137@tcp LPNI seq info [2081:2081:8:4294967295] 00000020:00000001:0.0:1784948763.061392:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948763.061394:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948763.061399:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948763.061403:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000263f8d59. 02000000:00000001:0.0:1784948763.061409:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:2.0:1784948763.061409:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:0.0:1784948763.061412:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948763.061415:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000049907b7f. 00000800:00000200:2.0:1784948763.061419:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948763.061420:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948763.061423:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784948763.061424:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000230b350e. 00000100:00000001:0.0:1784948763.061425:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948763.061430:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:2.0:1784948763.061431:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948763.061432:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948763.061436:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1784948763.061439:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:0.0:1784948763.061440:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948763.061445:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000230b350e type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948763.061457:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948763.061459:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948763.061460:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948763.061461:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948763.061463:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948763.061466:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005e465757 x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:424/0 lens 224/224 e 0 to 0 dl 1784948774 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948763.061473:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005e465757 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289650944:12345-192.168.202.137@tcp:400: Request processed in 323us (599us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948763.061478:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 674 00000100:00000040:2.0:1784948763.061480:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948763.061481:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948763.061482:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948763.061485:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000734334de. 00000020:00000010:2.0:1784948763.061488:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948763.061490:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f1904ca. 00000020:00000040:2.0:1784948763.061492:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948763.061494:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948763.061533:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784948763.061533:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948763.061538:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000230b350e. 00000800:00000001:2.0:1784948763.061541:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948763.061542:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948763.061547:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948763.061552:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948763.061552:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948763.061558:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000200:2.0:1784948763.061562:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:3.0:1784948763.061563:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000100:00000001:3.0:1784948763.061568:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948763.061570:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948763.061573:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435700 00000400:00000200:2.0:1784948763.061577:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8239 [1] + 224 00000800:00000001:2.0:1784948763.061581:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948763.061590:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948763.061592:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948763.061594:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948763.061597:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948763.061599:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948763.061605:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948763.061615:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948763.061619:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948763.061622:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948763.061627:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948763.061630:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948763.061633:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948763.061634:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948763.061636:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e686bcb0. 00000400:00000200:3.0:1784948763.061640:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:3.0:1784948763.061641:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:3.0:1784948763.061643:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948763.061646:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948763.061652:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948763.061657:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948763.061659:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948763.061660:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948763.061661:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948763.061662:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948763.061663:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948763.061665:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948763.061668:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=80 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948763.061670:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=80 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948763.061672:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948763.061673:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948763.061674:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948763.061676:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948763.061680:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948763.061682:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948763.061683:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948763.061684:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948763.061685:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948763.061687:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948763.061691:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948763.061692:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948763.061693:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948763.061695:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948763.061699:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005db4bd4e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289650944:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948763.061702:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948763.061703:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948763.061704:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005db4bd4e x1871651289650944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948763.061708:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948763.061709:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948763.061710:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:3.0:1784948763.061713:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948763.061714:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948763.061715:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948763.061716:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948763.061717:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000016632059. 02000000:00000001:3.0:1784948763.061719:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948763.061720:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948763.061721:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005db4bd4e. 00000100:00000001:3.0:1784948763.061722:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948763.061723:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948763.061724:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948763.061727:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948763.061727:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948763.061728:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948763.061730:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948763.102998:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305702442, at 4305702443 00000100:00000040:1.0:1784948763.102999:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305702442, at 4305702443 00000100:00000040:3.0:1784948763.103005:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305702442, at 4305702443 00000100:00000010:2.0:1784948763.103012:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:1.0:1784948763.103015:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000afdabce. 00000100:00000010:3.0:1784948763.103017:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:0.0:1784948763.103037:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305702442, at 4305702443 00000100:00000010:0.0:1784948763.103047:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00010000:00000001:2.0:1784948763.892015:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948763.892022:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948763.892029:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948763.892031:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948763.892105:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948763.892109:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948763.892111:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948764.083979:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948764.083991:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948764.083992:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948764.083996:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948764.083998:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948764.084001:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948764.084002:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948764.916043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948764.916051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948764.916058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948764.916059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948764.916121:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948764.916127:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948764.916130:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948765.108061:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948765.108073:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948765.108074:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948765.108078:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948765.108081:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948765.108084:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948765.108086:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948765.940085:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948765.940097:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948765.940108:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948765.940110:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948765.940238:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948765.940245:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948765.940251:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948766.132144:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948766.132174:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948766.132177:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948766.132184:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948766.132188:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948766.132193:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948766.132196:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948766.324023:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000771e09fe. 00000400:00000010:3.0:1784948766.324041:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002026f8a1. 00000400:00000010:3.0:1784948766.324046:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007f814b08. 00000400:00000010:3.0:1784948766.324052:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001542e92a. 00000400:00000010:3.0:1784948766.324056:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007864ed56. 00000400:00000010:3.0:1784948766.324060:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000541952f4. 00000400:00000010:3.0:1784948766.324064:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007f70671d. 00000400:00000010:3.0:1784948766.324068:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009c70d9ae. 00000400:00000010:3.0:1784948766.324074:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000029a19359. 00000400:00000010:3.0:1784948766.324077:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003ad675ec. 00000400:00000010:3.0:1784948766.324079:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000dbce1d1e. 00000400:00000010:3.0:1784948766.324082:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008fec2664. 00000400:00000010:3.0:1784948766.324085:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007834bcf1. 00000400:00000010:3.0:1784948766.324087:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a9029990. 00000400:00000010:3.0:1784948766.324089:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bfbf2878. 00000400:00000010:3.0:1784948766.324091:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000073e0dd6a. 00000400:00000010:3.0:1784948766.324094:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006fba1cc1. 00000400:00000010:3.0:1784948766.324096:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d6af1288. 00000400:00000010:3.0:1784948766.324098:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000073805bac. 00000400:00000010:3.0:1784948766.324101:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000c1ba2e6. 00000400:00000010:3.0:1784948766.324103:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bb67e12b. 00000400:00000010:3.0:1784948766.324105:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008c12d7f6. 00000400:00000010:3.0:1784948766.324107:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000341744ce. 00000400:00000010:3.0:1784948766.324109:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002cc5a29e. 00000400:00000010:3.0:1784948766.324111:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004429599f. 00000400:00000010:3.0:1784948766.324114:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000075884760. 00000400:00000010:3.0:1784948766.324116:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000058ef8883. 00000400:00000010:3.0:1784948766.324119:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009e0071c7. 00000400:00000010:3.0:1784948766.324125:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000041ec6752. 00000400:00000010:3.0:1784948766.324128:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f65f2fb7. 00000400:00000010:3.0:1784948766.324130:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008a1e1d5c. 00000400:00000010:3.0:1784948766.324138:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003b463cff. 00000400:00000010:3.0:1784948766.324140:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000091a157d3. 00000400:00000010:3.0:1784948766.324143:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006d2c4b24. 00000400:00000010:3.0:1784948766.324145:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007701057d. 00000400:00000010:3.0:1784948766.324147:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cc444ae0. 00010000:00000001:2.0:1784948766.964065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948766.964075:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948766.964084:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948766.964086:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948766.964182:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948766.964188:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948766.964193:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948767.156045:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948767.156055:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784948767.156071:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948742/real 1784948742] req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948767 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948767.156098:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948767.156101:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948767.156104:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000400:00000200:1.0:1784948767.156114:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003dcd3ddc 00000400:00000010:1.0:1784948767.156118:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003dcd3ddc. 00000100:00000001:1.0:1784948767.156123:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948767.156127:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948767 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948767.156138:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948767 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948767.156147:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948767.156150:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948767.156153:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948767.156156:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948767.156161:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948767 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948767.156170:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948767.156181:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948767.156185:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948767.156189:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948767.156194:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948767.156198:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948767.156207:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948767 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948767.156223:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784948767.156229:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948767.156232:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948767.156234:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948767.156240:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784948767.156244:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784948767.156251:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784948767.156291:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:1.0:1784948767.156298:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784948767.156306:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948767 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948767.156321:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289649792:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784948767.156329:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948767.156333:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948767.156337:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005dccb977 x1871651289649792/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948767 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948767.156348:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948767.156353:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1784948767.156354:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:1.0:1784948767.156358:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d8bbc1a8. 00000100:00000001:2.0:1784948767.156370:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:1.0:1784948767.156372:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948767.156380:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000010:2.0:1784948767.156381:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000078cad088. 00000020:00000040:1.0:1784948767.156387:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000040:2.0:1784948767.156391:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784948767.156394:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948767.156401:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 0000000077ad1613. 00000100:00000001:2.0:1784948767.156401:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948767.156409:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948767.156413:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948767.156417:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948767.156419:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948767.156425:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005dccb977. 02000000:00000010:2.0:1784948767.156433:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000006d8f9877. 00000100:00000001:1.0:1784948767.156434:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948767.156441:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948767.156447:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.156447:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948767.156458:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948767.156463:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948767.156464:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948767.156473:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948767.156479:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.156493:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948767.156497:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948767.156505:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000078cad088] to pc [ptlrpcd_00_03+3] req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948767.156551:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948767.156555:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948767.156559:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784948767.156563:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1784948767.156566:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784948767.156571:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1784948767.156577:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11014 00000400:00000200:2.0:1784948767.156591:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:2.0:1784948767.156599:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11013 00000100:00000001:1.0:1784948767.156611:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948767.156620:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000400:00000200:2.0:1784948767.156623:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:1.0:1784948767.156627:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948767.156633:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:1.0:1784948767.156636:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1784948767.156641:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948767.156643:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.156650:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784948767.156652:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948767.156657:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948767.156658:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784948767.156660:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948767.156666:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948767.156669:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:0.0:1784948767.156672:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.156676:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:1.0:1784948767.156678:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784948767.156684:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:2.0:1784948767.156692:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948767.156700:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:1.0:1784948767.156701:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948767.156708:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.156709:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784948767.156717:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1784948767.156717:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784948767.156726:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948767.156730:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000001:1.0:1784948767.156734:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.156735:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1784948767.156740:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00100000:1.0:1784948767.156745:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289651072:0@lo:400: 00000100:00000040:2.0:1784948767.156746:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:2.0:1784948767.156756:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:0.0:1784948767.156761:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000001:1.0:1784948767.156762:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1784948767.156764:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:1.0:1784948767.156774:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1784948767.156779:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000001:1.0:1784948767.156781:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948767.156784:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eff166f6. 02000000:00000001:1.0:1784948767.156791:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1784948767.156791:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000010:0.0:1784948767.156800:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000001:2.0:1784948767.156800:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948767.156801:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000461737cb. 02000000:00000001:2.0:1784948767.156807:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1784948767.156814:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948767.156820:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948767.156825:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000eed58f44. 02000000:00000001:2.0:1784948767.156827:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948767.156829:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000007e48c90. 00000100:00000001:2.0:1784948767.156837:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1784948767.156842:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948767.156843:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000200:0.0:1784948767.156854:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000010:2.0:1784948767.156857:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007676ae3f. 00000020:00000040:2.0:1784948767.156864:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:1.0:1784948767.156870:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289651072, portal 25 02000000:00000001:2.0:1784948767.156870:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1784948767.156876:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948767.156879:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948767.156882:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784948767.156942:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470018816 : -115544239532800 : ffff96e9c2a59500) 00000100:00000040:1.0:1784948767.156954:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948767.156961:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:1.0:1784948767.156978:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784948767.156983:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1784948767.156985:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948767.156993:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000400:00000200:0.0:1784948767.156995:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000200:1.0:1784948767.157003:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289651072, offset 0 00000400:00000200:0.0:1784948767.157008:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948767.157017:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000400:00000200:1.0:1784948767.157020:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948767.157033:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3206:3206:241:4294967295] 1.2.3.4@tcp LPNI seq info [95:95:8:4294967295] 00000400:00000200:0.0:1784948767.157044:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000400:00000200:0.0:1784948767.157055:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:0.0:1784948767.157065:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 02000000:00000001:2.0:1784948767.157076:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1784948767.157077:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784948767.157084:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948767.157096:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000901ee2b9. 00000400:00000200:1.0:1784948767.157104:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435780 00000100:00000001:2.0:1784948767.157107:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948767.157124:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784948767.157125:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7835 [2] + 0 00000400:00000010:1.0:1784948767.157139:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000010aeeaab. 00000400:00000200:1.0:1784948767.157151:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000096ccf04d 00000100:00000001:2.0:1784948767.157152:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948767.157158:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948767.157168:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007676ae3f] to pc [ptlrpcd_rcv+-1] req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1784948767.157169:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d1b12746. 00000400:00000200:1.0:1784948767.157169:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948767.157191:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948767.157202:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096ccf04d 00000100:00000001:2.0:1784948767.157203:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948767.157208:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:1.0:1784948767.157211:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000096ccf04d. 00000100:00000040:2.0:1784948767.157212:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1784948767.157216:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 36 secs later) 00000100:00000001:2.0:1784948767.157226:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948767.157230:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000010:2.0:1784948767.157233:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086d6bb31. 00000100:00000200:1.0:1784948767.157239:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.1:1784948767.157240:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000020:00000040:2.0:1784948767.157243:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000800:00000200:0.1:1784948767.157254:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 36 secs later) 00000100:00000001:2.0:1784948767.157255:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1784948767.157256:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000bcf1bd0a x1871651289651072 msgsize 224 00000100:00000001:3.0:1784948767.157260:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784948767.157261:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:0.0:1784948767.157271:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 02000000:00000001:2.0:1784948767.157272:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948767.157274:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000010:0.0:1784948767.157280:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d1b12746. 00000100:00000001:3.0:1784948767.157284:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:2.0:1784948767.157286:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c03b4f3a. 00000100:00100000:1.0:1784948767.157290:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:0.0:1784948767.157291:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1784948767.157295:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948767.157299:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:2.0:1784948767.157310:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784948767.157311:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000001:1.0:1784948767.157319:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948767.157325:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000001:2.0:1784948767.157334:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1784948767.157336:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000200:0.0:1784948767.157339:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000001:2.0:1784948767.157339:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948767.157342:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000100:00000001:1.0:1784948767.157356:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784948767.157360:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000040:2.0:1784948767.157363:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086d6bb31] to pc [ptlrpcd_00_00+0] req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948767.157365:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948767.157368:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948767.157379:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000400:00000200:0.0:1784948767.157394:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000400:00000200:0.0:1784948767.157413:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:0.0:1784948767.157426:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948767.157431:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948767.157438:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:0.0:1784948767.157445:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:1.0:1784948767.157449:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948767.157450:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784948767.157452:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:2.0:1784948767.157452:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948767.157457:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948767.157460:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948767.157466:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000040:2.0:1784948767.157467:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11044) 02000000:00000001:3.0:1784948767.157469:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948767.157477:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948767.157481:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eff166f6. 02000000:00000001:3.0:1784948767.157489:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948767.157497:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:2.0:1784948767.157503:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948767.157504:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948767.157505:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007676ae3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289651200:1.2.3.4@tcp:8: 00000400:00000200:0.0:1784948767.157510:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00100000:2.0:1784948767.157511:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289651072 00000100:00000001:1.0:1784948767.157515:0:337121:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784948767.157521:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948767.157526:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784948767.157528:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948767.157531:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948767.157533:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.157535:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948767.157541:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:3.0:1784948767.157545:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948767.157546:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305706539 02000000:00000001:2.0:1784948767.157548:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948767.157557:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948767.157560:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289651072 00000100:00000001:3.0:1784948767.157563:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784948767.157564:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000c54be64. 00000020:00000001:2.0:1784948767.157565:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948767.157572:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784948767.157577:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948767.157581:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948767.157582:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000040:2.0:1784948767.157582:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948767.157590:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948767.157594:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948767.157597:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948767.157601:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948767.157602:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305706539 00000100:00000001:3.0:1784948767.157603:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948767.157609:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948767.157611:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948767.157612:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948767.157615:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000092889e7d. 00000100:00100000:0.0:1784948767.157618:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000020:00000010:2.0:1784948767.157621:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006b18ba8f. 00000100:00000001:3.0:1784948767.157623:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948767.157627:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948767.157628:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948767.157632:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948767.157641:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006b18ba8f. 00000100:00000001:1.0:1784948767.157644:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305706539 00000100:00000001:0.0:1784948767.157646:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784948767.157650:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000100:00000001:0.0:1784948767.157654:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784948767.157657:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:0.0:1784948767.157662:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1784948767.157662:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e42ee6ac. 00000100:00000001:1.0:1784948767.157665:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948767.157670:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948767.157676:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000010:2.0:1784948767.157678:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000001:1.0:1784948767.157679:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305706539 00000100:00000040:0.0:1784948767.157687:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784948767.157692:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000040:2.0:1784948767.157698:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948767.157704:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948767.157708:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948767.157715:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948767.157717:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948767.157723:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948767.157725:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948767.157726:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948767.157732:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948767.157735:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.157737:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948767.157739:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948767.157742:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948767.157747:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948767.157753:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948767.157757:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289651328:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948767.157760:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948767.157766:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948767.157773:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948767.157776:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948767.157779:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948767.157783:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000010:0.0:1784948767.157786:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eff166f6. 00000100:00000001:1.0:1784948767.157788:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948767.157789:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:0.0:1784948767.157793:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948767.157796:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948767.157800:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001c023af. 00000100:00100000:2.0:1784948767.157806:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 699 00000400:00000010:0.0:1784948767.157811:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000040:2.0:1784948767.157814:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000200:0.0:1784948767.157821:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289651328, portal 10 00000100:00000001:2.0:1784948767.157821:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180531280 : -115539529020336 : ffff96eadb6a4c50) 00000100:00000001:0.0:1784948767.157834:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948767.157837:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470019712 : -115544239531904 : ffff96e9c2a59880) 00000100:00000040:2.0:1784948767.157839:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bcf1bd0a x1871651289651072/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:428/0 lens 224/0 e 0 to 0 dl 1784948778 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948767.157854:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948767.157974:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948767.157977:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948767.157985:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006b18ba8f. 00000100:00000040:0.0:1784948767.157986:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948767.157994:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948767.157995:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000020:00000010:2.0:1784948767.158001:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006b18ba8f. 00000100:00000200:0.0:1784948767.158003:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289651328, offset 0 00000100:00100000:2.0:1784948767.158018:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bcf1bd0a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289651072:12345-0@lo:400: 00000400:00000200:0.0:1784948767.158019:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000200:2.0:1784948767.158031:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289651072 00000020:00000001:2.0:1784948767.158037:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784948767.158038:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:2.0:1784948767.158044:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784948767.158049:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1784948767.158051:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.158057:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:0.0:1784948767.158058:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:2.0:1784948767.158063:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:0.0:1784948767.158067:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1784948767.158070:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:0.0:1784948767.158076:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:2.0:1784948767.158077:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948767.158085:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:0.0:1784948767.158091:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3207:3207:241:4294967295] 192.168.202.137@tcp LPNI seq info [2082:2082:8:4294967295] 00000020:00000001:2.0:1784948767.158092:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948767.158098:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.158103:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948767.158109:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948767.158114:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:0.0:1784948767.158115:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000010:2.0:1784948767.158122:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000004b76201b. 00000800:00000200:0.0:1784948767.158127:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948767.158130:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.158136:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948767.158137:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d1b12746. 00000020:00000001:2.0:1784948767.158140:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948767.158146:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289651072 00000800:00000200:0.0:1784948767.158153:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00010000:00000001:2.0:1784948767.158155:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948767.158164:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bcf1bd0a x1871651289651072/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:428/0 lens 224/224 e 0 to 0 dl 1784948778 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948767.158174:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948767.158185:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d1b12746 type 1, nob 320 niov 1 nkiov 1 00010000:00000001:2.0:1784948767.158191:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948767.158196:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948767.158200:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=89 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948767.158205:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948767.158208:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948767.158211:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.158213:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784948767.158221:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:0.0:1784948767.158222:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2040 usecs (3 jiffies) 02000000:00000001:2.0:1784948767.158224:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948767.158235:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305706501 00000100:00000001:2.0:1784948767.158236:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948767.158245:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305706539 00000100:00000040:2.0:1784948767.158246:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948767.158255:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305706501 00000400:00000010:2.0:1784948767.158257:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000100:00000001:0.0:1784948767.158259:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305706539 00000100:00000200:2.0:1784948767.158264:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289651072, offset 224 00000100:00000001:0.0:1784948767.158265:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305706501 00000100:00000001:0.0:1784948767.158275:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305706539 00000100:00000001:0.0:1784948767.158279:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305706501 00000400:00000200:2.0:1784948767.158284:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948767.158285:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305706539 00000100:00000001:0.0:1784948767.158293:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948767.158296:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948767.158300:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948767.158305:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948767.158308:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948767.158324:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435780 00000400:00000200:2.0:1784948767.158334:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8251 [1] + 224 00000400:00000200:2.0:1784948767.158342:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948767.158347:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948767.158352:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948767.158355:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948767.158367:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948767.158392:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784948767.158400:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:2.0:1784948767.158403:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000100:00000001:2.0:1784948767.158406:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948767.158408:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948767.158415:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948767.158419:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784948767.158422:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948767.158426:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.158429:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000800:00000200:1.0:1784948767.158430:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:2.0:1784948767.158436:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bcf1bd0a x1871651289651072/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:428/0 lens 224/224 e 0 to 0 dl 1784948778 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1784948767.158440:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d1b12746. 00000400:00000200:1.0:1784948767.158451:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:2.0:1784948767.158459:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bcf1bd0a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289651072:12345-0@lo:400: Request processed in 478us (1198us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948767.158467:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:0.0:1784948767.158468:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948767.158471:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948767.158478:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948767.158488:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000800:00000001:0.0:1784948767.158490:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948767.158490:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 699 00000400:00000010:1.0:1784948767.158496:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000800:00000001:0.0:1784948767.158501:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784948767.158504:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948767.158507:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:3.0:1784948767.158508:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784948767.158510:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000001:0.0:1784948767.158511:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1784948767.158517:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784948767.158518:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:1.0:1784948767.158521:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948767.158525:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000eed58f44. 00000020:00000010:2.0:1784948767.158528:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e42ee6ac. 00000400:00000200:0.0:1784948767.158537:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948767.158542:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000020:00000010:2.0:1784948767.158543:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000400:00000010:3.0:1784948767.158553:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000020:00000010:2.0:1784948767.158557:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000100:00000001:3.0:1784948767.158565:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1784948767.158569:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:0.0:1784948767.158572:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435880 00000100:00000001:1.0:1784948767.158573:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:3.0:1784948767.158580:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948767.158581:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000040:1.0:1784948767.158587:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784948767.158589:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000004b76201b. 00000400:00000200:0.0:1784948767.158596:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 152240 02000000:00000001:2.0:1784948767.158601:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784948767.158608:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948767.158623:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:2.0:1784948767.158624:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:2.0:1784948767.158639:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000045b97c3a. 00000800:00000001:0.0:1784948767.158640:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948767.158655:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948767.158656:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948767.158656:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000be5664ea. 00000100:00000200:3.0:1784948767.158661:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948767.158670:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948767.158679:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948767.158694:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948767.158698:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948767.158700:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948767.158704:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784948767.158707:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948767.158714:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000010:0.0:1784948767.158715:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000084bed21d. 00000100:00000001:3.0:1784948767.158718:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:0.0:1784948767.158723:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000084bed21d x1871651289651328 msgsize 224 00000100:00000001:3.0:1784948767.158724:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948767.158735:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948767.158744:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00001000:3.0:1784948767.158747:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=89 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948767.158754:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=89 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948767.158758:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948767.158760:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948767.158763:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784948767.158766:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948767.158767:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948767.158771:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948767.158780:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948767.158780:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948767.158789:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948767.158801:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948767.158806:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948767.158809:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948767.158813:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948767.158824:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289651072:0@lo:400: 00000100:00000001:2.0:1784948767.158831:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948767.158833:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948767.158839:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289651328 00000100:00000001:3.0:1784948767.158840:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1784948767.158847:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:3.0:1784948767.158847:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000078cad088 x1871651289651072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948767.158853:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948767.158859:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948767.158869:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948767.158871:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948767.158877:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1784948767.158879:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289651328 02000000:00000010:3.0:1784948767.158881:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000461737cb. 00000020:00000001:2.0:1784948767.158931:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948767.158935:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784948767.158937:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948767.158941:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784948767.158944:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948767.158948:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948767.158952:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948767.158957:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006b18ba8f. 00000100:00000001:2.0:1784948767.158966:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948767.158966:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948767.158974:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:2.0:1784948767.158975:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006b18ba8f. 00000020:00000040:3.0:1784948767.158981:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000010:2.0:1784948767.158986:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000020:00000001:3.0:1784948767.158987:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948767.158993:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000006d8f9877. 00000020:00000010:2.0:1784948767.158996:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e42ee6ac. 02000000:00000001:3.0:1784948767.159001:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000010:2.0:1784948767.159004:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 02000000:00000001:3.0:1784948767.159007:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948767.159012:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000078cad088. 00000100:00000040:2.0:1784948767.159017:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948767.159019:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948767.159024:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.159026:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948767.159030:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.159032:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948767.159039:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948767.159043:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948767.159046:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948767.159048:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.159053:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948767.159055:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948767.159058:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948767.159061:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.159076:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.159091:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948767.159092:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948767.159098:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 675 00000100:00000040:2.0:1784948767.159102:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948767.159105:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558298496 : -115544151253120 : ffff96e9c7e89f80) 00000100:00000040:2.0:1784948767.159112:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000084bed21d x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:428/0 lens 224/0 e 0 to 0 dl 1784948778 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948767.159122:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948767.159126:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006b18ba8f. 00000100:00000001:2.0:1784948767.159129:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948767.159132:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006b18ba8f. 00000100:00100000:2.0:1784948767.159139:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000084bed21d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289651328:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784948767.159144:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289651328 00000020:00000001:2.0:1784948767.159147:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948767.159150:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948767.159152:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.159154:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948767.159157:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784948767.159160:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948767.159163:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948767.159165:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948767.159167:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948767.159169:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.159171:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948767.159175:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948767.159176:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948767.159180:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000004b76201b. 02000000:00000001:2.0:1784948767.159183:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.159185:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.159187:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948767.159190:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289651328 00010000:00000001:2.0:1784948767.159193:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948767.159196:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000084bed21d x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:428/0 lens 224/224 e 0 to 0 dl 1784948778 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948767.159206:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948767.159209:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948767.159211:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=84 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948767.159215:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948767.159218:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948767.159220:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948767.159223:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948767.159225:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.159228:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948767.159231:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948767.159234:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000100:00000200:2.0:1784948767.159237:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289651328, offset 224 00000400:00000200:2.0:1784948767.159242:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948767.159251:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948767.159257:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3208:3208:241:4294967295] 192.168.202.137@tcp LPNI seq info [2083:2083:8:4294967295] 00000400:00000200:2.0:1784948767.159269:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948767.159274:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948767.159278:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006aefc2f9. 00000800:00000200:2.0:1784948767.159284:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948767.159291:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948767.159294:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000006aefc2f9 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948767.159312:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948767.159316:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948767.159320:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948767.159322:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948767.159324:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948767.159328:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000084bed21d x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:428/0 lens 224/224 e 0 to 0 dl 1784948778 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948767.159339:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000084bed21d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289651328:12345-192.168.202.137@tcp:400: Request processed in 215us (615us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948767.159347:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 675 00000100:00000040:2.0:1784948767.159350:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948767.159352:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948767.159354:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948767.159358:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e42ee6ac. 00000020:00000010:2.0:1784948767.159361:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948767.159364:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000020:00000040:2.0:1784948767.159368:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948767.159370:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948767.159527:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948767.159535:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006aefc2f9. 00000400:00000200:0.0:1784948767.159540:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948767.159547:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948767.159553:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:0.0:1784948767.159555:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000100:00000001:0.0:1784948767.159559:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948767.159561:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948767.159568:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948767.159574:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948767.159578:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948767.159581:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948767.159589:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948767.159601:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435880 00000400:00000200:0.0:1784948767.159610:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8259 [1] + 224 00000800:00000001:0.0:1784948767.159619:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948767.159628:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948767.159631:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948767.159635:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948767.159639:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948767.159644:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948767.159657:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948767.159683:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948767.159689:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948767.159692:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.159801:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948767.159807:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948767.159814:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948767.159816:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948767.159819:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000001c023af. 00000400:00000200:2.0:1784948767.159827:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:2.0:1784948767.159829:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:2.0:1784948767.159833:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948767.159841:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948767.159853:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948767.159864:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948767.159865:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.159868:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948767.159870:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948767.159872:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948767.159875:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948767.159878:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948767.159940:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=84 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948767.159946:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=84 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948767.159949:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948767.159950:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948767.159953:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948767.159958:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948767.159967:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.159971:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948767.159973:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948767.159976:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948767.159977:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948767.159981:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948767.159989:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948767.159991:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948767.159993:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948767.159997:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948767.160006:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289651328:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948767.160011:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.160014:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948767.160016:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000086d6bb31 x1871651289651328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948783 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948767.160023:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948767.160025:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948767.160028:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eff166f6. 02000000:00000001:2.0:1784948767.160031:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948767.160033:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948767.160035:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948767.160039:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948767.160041:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c03b4f3a. 02000000:00000001:2.0:1784948767.160044:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948767.160045:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948767.160047:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000086d6bb31. 00000100:00000001:2.0:1784948767.160051:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948767.160052:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.160054:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948767.160060:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948767.160061:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948767.160064:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948767.160066:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948767.200042:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305706539, at 4305706540 00000100:00000040:3.0:1784948767.200053:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305706539, at 4305706540 00000100:00000010:1.0:1784948767.200066:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000092889e7d. 00000100:00000040:2.0:1784948767.200068:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305706539, at 4305706540 00000100:00000040:0.0:1784948767.200071:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305706539, at 4305706540 00000100:00000010:3.0:1784948767.200075:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000010:2.0:1784948767.200088:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:0.0:1784948767.200092:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000c54be64. 00010000:00000001:2.0:1784948767.987998:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948767.988004:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948767.988008:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948767.988010:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948767.988054:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948767.988058:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948767.988060:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948768.180044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948768.180054:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1784948768.180072:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948767/real 0] req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948768 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948768.180106:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.180110:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948768.180112:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948768.180116:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948768 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948768.180125:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.180135:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948768.180137:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.180140:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948768.180143:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948768.180146:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948768.180150:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948768 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948768.180168:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1784948768.180172:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.180173:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948768.180175:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.180177:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1784948768.180179:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1784948768.180183:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1784948768.180208:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:3.0:1784948768.180212:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1784948768.180216:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948768 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948768.180240:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007676ae3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289651200:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784948768.180246:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.180249:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948768.180252:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007676ae3f x1871651289651200/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948768 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948768.180259:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948768.180259:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1784948768.180262:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948768.180266:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:2.0:1784948768.180267:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000020:00000001:3.0:1784948768.180270:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000010:2.0:1784948768.180274:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086d6bb31. 02000000:00000010:3.0:1784948768.180274:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000901ee2b9. 00000020:00000040:2.0:1784948768.180282:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 02000000:00000001:3.0:1784948768.180283:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948768.180287:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1784948768.180288:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:3.0:1784948768.180290:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007676ae3f. 02000000:00000001:2.0:1784948768.180292:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948768.180296:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1784948768.180298:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.180298:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.180304:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.180306:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:2.0:1784948768.180307:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c03b4f3a. 00000100:00000001:3.0:1784948768.180308:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.180314:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948768.180316:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.180317:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948768.180325:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948768.180333:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948768.180335:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948768.180340:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086d6bb31] to pc [ptlrpcd_00_01+1] req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948768.180376:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948768.180379:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948768.180382:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784948768.180385:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1784948768.180386:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948768.180390:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948768.180390:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1784948768.180395:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11014 00000100:00000001:0.0:1784948768.180396:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948768.180400:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948768.180403:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948768.180407:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948768.180408:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:3.0:1784948768.180408:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948768.180416:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11039 00000100:00000001:0.0:1784948768.180425:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948768.180427:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948768.180430:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948768.180432:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784948768.180435:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 02000000:00000001:0.0:1784948768.180437:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948768.180441:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00100000:0.0:1784948768.180444:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289651456:0@lo:400: 00000100:00080000:2.0:1784948768.180445:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1784948768.180448:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:0.0:1784948768.180453:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1784948768.180453:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000400:00000200:3.0:1784948768.180456:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 02000000:00000001:0.0:1784948768.180459:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784948768.180460:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784948768.180464:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948768.180465:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:0.0:1784948768.180468:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:3.0:1784948768.180468:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000208850ab. 00000100:00000001:2.0:1784948768.180469:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000010:0.0:1784948768.180474:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 00000020:00000001:2.0:1784948768.180475:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:3.0:1784948768.180477:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 02000000:00000001:0.0:1784948768.180479:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948768.180479:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:2.0:1784948768.180484:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784948768.180488:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000948ad730. 00000020:00000040:2.0:1784948768.180488:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:3.0:1784948768.180489:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000279bae33. 00000020:00000001:2.0:1784948768.180494:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:2.0:1784948768.180499:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:3.0:1784948768.180499:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000040:2.0:1784948768.180522:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:2.0:1784948768.180527:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1784948768.180530:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000100:00000040:2.0:1784948768.180531:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000400:00000200:3.0:1784948768.180540:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:2.0:1784948768.180541:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000200:0.0:1784948768.180542:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289651456, portal 25 00000020:00000040:2.0:1784948768.180554:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:0.0:1784948768.180555:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1784948768.180557:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784948768.180562:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470019712 : -115544239531904 : ffff96e9c2a59880) 00000100:00080000:2.0:1784948768.180569:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000400:00000200:3.0:1784948768.180571:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:0.0:1784948768.180575:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1784948768.180587:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000400:00000200:3.0:1784948768.180588:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784948768.180600:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948768.180602:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:2.0:1784948768.180611:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1784948768.180614:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784948768.180619:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3209:3209:241:4294967295] 1.2.3.4@tcp LPNI seq info [96:96:8:4294967295] 00000100:00000040:0.0:1784948768.180627:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784948768.180628:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948768.180636:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000400:00000010:0.0:1784948768.180637:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 02000000:00000001:2.0:1784948768.180638:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948768.180649:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289651456, offset 0 00000100:00000001:2.0:1784948768.180649:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1784948768.180656:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948768.180657:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000400:00000200:0.0:1784948768.180667:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000010:2.0:1784948768.180669:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008d623923. 00000020:00000040:2.0:1784948768.180679:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000800:00000200:3.0:1784948768.180679:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000800:00000010:3.0:1784948768.180698:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000400:00000200:0.0:1784948768.180701:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784948768.180701:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1784948768.180708:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.180714:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948768.180717:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948768.180720:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948768.180725:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435900 00000800:00000200:3.0:1784948768.180730:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 02000000:00000010:2.0:1784948768.180733:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000009b7d615c. 00000800:00000200:3.0:1784948768.180736:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 35 secs later) 00000100:00000001:2.0:1784948768.180740:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948768.180743:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7859 [2] + 0 00000400:00000010:0.0:1784948768.180753:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000839265ff. 00000800:00000200:3.1:1784948768.180760:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00100000:2.0:1784948768.180761:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784948768.180763:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b3c5fd98 00000800:00000200:3.1:1784948768.180772:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 35 secs later) 00000400:00000200:0.0:1784948768.180778:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000100:3.0:1784948768.180790:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000400:00000200:0.0:1784948768.180795:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000010:3.0:1784948768.180798:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000100:00000001:2.0:1784948768.180818:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1784948768.180823:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b3c5fd98 00000100:00000001:2.0:1784948768.180823:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948768.180825:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000010:0.0:1784948768.180829:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b3c5fd98. 00000100:00000040:2.0:1784948768.180832:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d623923] to pc [ptlrpcd_rcv+-1] req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948768.180835:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:3.0:1784948768.180838:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000200:0.0:1784948768.180839:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948768.180845:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000012145ed7 x1871651289651456 msgsize 224 00000400:00000200:3.0:1784948768.180849:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00100000:0.0:1784948768.180856:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784948768.180859:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000040:0.0:1784948768.180861:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784948768.180861:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948768.180865:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000040:2.0:1784948768.180867:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948768.180872:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948768.180941:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000edab6e1. 00000020:00000040:2.0:1784948768.180946:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948768.180948:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948768.180948:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:2.0:1784948768.180951:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948768.180973:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1784948768.180977:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 02000000:00000001:2.0:1784948768.180980:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948768.180981:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:0.0:1784948768.180990:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000400:00000200:3.0:1784948768.180991:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 02000000:00000010:2.0:1784948768.180994:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000762a0fe3. 00000100:00000001:0.0:1784948768.181002:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948768.181006:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784948768.181010:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:0.0:1784948768.181013:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784948768.181016:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1784948768.181021:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1784948768.181024:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:3.0:1784948768.181032:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000400:00000200:3.0:1784948768.181044:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948768.181058:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:1.0:1784948768.181065:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948768.181068:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1784948768.181070:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000208850ab. 00000100:00000001:2.0:1784948768.181071:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948768.181093:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948768.181095:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948768.181096:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948768.181099:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1784948768.181103:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000040:2.0:1784948768.181109:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000edab6e1] to pc [ptlrpcd_00_02+2] req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948768.181111:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948768.181115:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000400:00000200:3.0:1784948768.181129:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:0.0:1784948768.181171:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948768.181171:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948768.181175:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.181176:0:337120:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1784948768.181182:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948768.181184:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948768.181186:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948768.181187:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948768.181194:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11045) 02000000:00000001:1.0:1784948768.181195:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.181198:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305707563 00000100:00100000:1.0:1784948768.181205:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289651584:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784948768.181209:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948768.181214:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:0.0:1784948768.181215:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 02000000:00000001:1.0:1784948768.181216:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948768.181222:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948768.181229:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006107fe19. 00000100:00000001:0.0:1784948768.181230:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1784948768.181234:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.181236:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948768.181242:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:1.0:1784948768.181243:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000010aeeaab. 00000100:00000001:0.0:1784948768.181246:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305707563 00000100:00000001:2.0:1784948768.181246:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:1.0:1784948768.181250:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000100:00000040:2.0:1784948768.181257:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784948768.181258:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000200:1.0:1784948768.181259:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289651584, portal 4 00000100:00000001:1.0:1784948768.181269:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948768.181272:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948768.181272:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470027776 : -115544239523840 : ffff96e9c2a5b800) 00000100:00000001:2.0:1784948768.181275:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948768.181281:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.181284:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305707563 02000000:00000001:2.0:1784948768.181286:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784948768.181290:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948768.181291:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1784948768.181296:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 02000000:00000001:2.0:1784948768.181299:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.181309:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00100000:2.0:1784948768.181311:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000edab6e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289651712:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948768.181314:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948768.181321:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948768.181323:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305707563 00000100:00000040:1.0:1784948768.181327:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948768.181328:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948768.181334:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948768.181335:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000400:00000010:1.0:1784948768.181337:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 02000000:00000001:2.0:1784948768.181338:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000200:1.0:1784948768.181344:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289651584, offset 0 02000000:00000010:2.0:1784948768.181355:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 00000100:00000001:0.0:1784948768.181357:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000200:1.0:1784948768.181364:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948768.181364:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.181369:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948768.181374:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.181374:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:2.0:1784948768.181375:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a0670fc. 00000400:00000200:1.0:1784948768.181381:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00100000:3.0:1784948768.181381:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289651456 00000100:00000001:0.0:1784948768.181384:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948768.181387:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 02000000:00000001:3.0:1784948768.181388:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948768.181392:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948768.181394:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948768.181395:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784948768.181397:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289651712, portal 10 00000100:00000001:0.0:1784948768.181402:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948768.181402:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.181403:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784948768.181406:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784948768.181408:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948768.181409:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470025984 : -115544239525632 : ffff96e9c2a5b100) 02000000:00000001:3.0:1784948768.181413:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948768.181416:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948768.181419:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784948768.181422:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289651456 00000100:00000001:0.0:1784948768.181424:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784948768.181428:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948768.181429:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:3.0:1784948768.181430:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948768.181431:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948768.181437:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948768.181440:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948768.181442:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784948768.181444:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3210:3210:241:4294967295] 192.168.202.137@tcp LPNI seq info [2084:2084:8:4294967295] 00000020:00000040:3.0:1784948768.181449:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:3.0:1784948768.181457:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948768.181460:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948768.181464:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948768.181467:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.181470:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948768.181477:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784948768.181478:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:2.0:1784948768.181485:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000010:3.0:1784948768.181486:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ea5f5298. 00000400:00000010:2.0:1784948768.181493:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000100:00000001:3.0:1784948768.181494:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1784948768.181495:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000200:2.0:1784948768.181500:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289651712, offset 0 00000020:00000010:3.0:1784948768.181504:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ea5f5298. 00000800:00000010:1.0:1784948768.181506:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000635133da. 00000400:00000200:2.0:1784948768.181513:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000800:00000200:1.0:1784948768.181518:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784948768.181525:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000800:00000200:1.0:1784948768.181532:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:2.0:1784948768.181535:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:1.0:1784948768.181540:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000635133da type 1, nob 616 niov 1 nkiov 1 00000400:00000200:2.0:1784948768.181544:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948768.181551:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948768.181554:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784948768.181558:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948768.181565:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3211:3211:240:4294967295] 192.168.202.137@tcp LPNI seq info [2085:2085:7:4294967295] 00000020:00000010:3.0:1784948768.181566:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000100:00000040:1.0:1784948768.181567:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:1.0:1784948768.181577:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305707563 00000020:00000010:3.0:1784948768.181579:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bb54a175. 00000100:00000001:1.0:1784948768.181583:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305707563 00000020:00000010:3.0:1784948768.181588:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:1.0:1784948768.181590:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305707563 00000400:00000200:2.0:1784948768.181597:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784948768.181599:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305707563 00000100:00000040:3.0:1784948768.181602:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948768.181608:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305707563 00000100:00000001:3.0:1784948768.181610:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000800:00000200:2.0:1784948768.181611:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948768.181614:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305707563 00000100:00000001:3.0:1784948768.181615:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948768.181621:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305707563 00000100:00000001:3.0:1784948768.181621:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784948768.181622:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d625f2e8. 00000100:00000001:1.0:1784948768.181631:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305707563 00000100:00000001:3.0:1784948768.181635:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948768.181637:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948768.181639:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948768.181647:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.181650:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948768.181652:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784948768.181653:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948768.181658:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948768.181660:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d625f2e8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948768.181665:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948768.181669:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1784948768.181678:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948768.181679:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948768.181685:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948768.181686:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948768.181687:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948768.181688:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948768.181696:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948768.181697:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2041 usecs (3 jiffies) 00000100:00100000:3.0:1784948768.181698:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 700 00000100:00000001:2.0:1784948768.181705:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305707524 00000100:00000040:3.0:1784948768.181709:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948768.181711:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305707563 00000800:00000200:0.0:1784948768.181712:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948768.181720:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305707524 00000100:00000001:3.0:1784948768.181720:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180539472 : -115539529012144 : ffff96eadb6a6c50) 00000800:00000010:0.0:1784948768.181721:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000635133da. 00000100:00000001:2.0:1784948768.181729:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305707563 00000400:00000200:0.0:1784948768.181732:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948768.181737:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305707524 00000100:00000040:3.0:1784948768.181739:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000012145ed7 x1871651289651456/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:429/0 lens 224/0 e 0 to 0 dl 1784948779 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948768.181744:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305707563 00000400:00000200:0.0:1784948768.181747:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948768.181753:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305707524 00000400:00000200:0.0:1784948768.181758:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0c9b918 00000100:00000001:2.0:1784948768.181761:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305707563 00000400:00000010:0.0:1784948768.181764:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f0c9b918. 00000100:00000001:2.0:1784948768.181769:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948768.181774:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948768.181794:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948768.181801:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948768.181804:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.181814:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948768.181815:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948768.181827:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ea5f5298. 00000100:00000001:0.0:1784948768.181828:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948768.181829:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948768.181836:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948768.181839:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948768.181840:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948793 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948768.181843:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948768.181845:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948768.181851:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1784948768.181855:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ea5f5298. 00000100:00000001:2.0:1784948768.181857:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.181867:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.181872:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.181897:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784948768.181903:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948768.181951:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000012145ed7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289651456:12345-0@lo:400: 00000100:00000200:3.0:1784948768.181958:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289651456 00000020:00000001:3.0:1784948768.181963:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948768.181969:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000001:1.0:1784948768.181970:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948768.181991:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948768.181996:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948768.181999:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784948768.182003:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948768.182006:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948768.182009:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948768.182012:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948768.182015:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948768.182018:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948768.182022:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948768.182026:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948768.182032:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 02000000:00000001:3.0:1784948768.182036:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.182040:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948768.182043:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948768.182044:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948768.182051:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289651456 00010000:00000001:3.0:1784948768.182055:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:2.0:1784948768.182057:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948768.182063:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948768.182064:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000200:3.0:1784948768.182064:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000012145ed7 x1871651289651456/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:429/0 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1784948768.182089:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d625f2e8. 00000800:00000001:2.0:1784948768.182090:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948768.182097:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948768.182103:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784948768.182104:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:2.0:1784948768.182108:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:3.0:1784948768.182110:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948768.182113:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00001000:3.0:1784948768.182120:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=90 v=5 (1 1 1 1) 00000400:00000200:1.0:1784948768.182124:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948768.182127:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000100:00000001:3.0:1784948768.182128:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1784948768.182133:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000400:00000200:2.0:1784948768.182134:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435a00 00000100:00000040:3.0:1784948768.182137:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948768.182142:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948768.182145:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:1.0:1784948768.182147:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6435980 00000400:00000200:2.0:1784948768.182154:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 152464 00000100:00000200:0.0:1784948768.182155:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948768.182155:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948768.182160:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1784948768.182175:0:337114:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289651584 offset 0 length 520: 4 00000800:00000001:2.0:1784948768.182179:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948768.182180:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948768.182180:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784948768.182187:0:337114:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000002d91eb54 is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000040:0.0:1784948768.182193:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948768.182194:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:1.0:1784948768.182195:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:2.0:1784948768.182197:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948768.182204:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000400:00000200:2.0:1784948768.182206:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948768.182211:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784948768.182214:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289651456, offset 224 00000100:00000001:0.0:1784948768.182219:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948768.182220:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948768.182224:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784948768.182228:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948768.182229:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:1.0:1784948768.182237:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948768.182240:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000800:00000001:1.0:1784948768.182243:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948768.182245:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:3.0:1784948768.182250:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000010:2.0:1784948768.182254:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007676ae3f. 00000100:00000040:2.0:1784948768.182259:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000007676ae3f x1871651289651712 msgsize 224 00000400:00000200:3.0:1784948768.182266:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435900 00000100:00100000:2.0:1784948768.182275:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:3.0:1784948768.182280:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8271 [1] + 224 00000400:00000200:3.0:1784948768.182288:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948768.182290:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948768.182295:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:2.0:1784948768.182302:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948768.182305:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:2.0:1784948768.182310:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948768.182314:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948768.182322:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948768.182326:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289651712 00000100:00000040:3.0:1784948768.182327:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948768.182332:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948768.182336:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948768.182340:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948768.182346:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948768.182349:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289651712 00000020:00000001:0.0:1784948768.182351:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948768.182353:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948768.182355:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948768.182355:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1784948768.182368:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000020:00000040:0.0:1784948768.182370:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000010:3.0:1784948768.182372:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000020:00000001:0.0:1784948768.182393:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948768.182397:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:0.0:1784948768.182400:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948768.182400:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948768.182402:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948768.182406:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948768.182412:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948768.182413:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784948768.182416:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:0.0:1784948768.182419:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000400:00000001:2.0:1784948768.182421:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1784948768.182422:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948768.182429:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784948768.182430:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000948ad730. 00010000:00000001:3.0:1784948768.182431:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948768.182436:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948768.182441:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000020:00000001:3.0:1784948768.182442:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:2.0:1784948768.182444:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:2.0:1784948768.182450:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000020:00000010:0.0:1784948768.182452:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000069d2b91d. 00000100:00000040:3.0:1784948768.182456:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000012145ed7 x1871651289651456/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:429/0 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948768.182457:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784948768.182462:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002632f5ee. 00000100:00000200:2.0:1784948768.182470:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948768.182474:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948768.182489:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948768.182496:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948768.182502:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948768.182509:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948768.182514:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000012145ed7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289651456:12345-0@lo:400: Request processed in 696us (1660us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948768.182523:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:2.0:1784948768.182525:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948768.182529:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948768.182535:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948768.182536:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 700 00000100:00000040:3.0:1784948768.182548:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948768.182555:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948768.182562:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.182562:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784948768.182569:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948768.182573:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948768.182573:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bb54a175. 00000100:00000001:0.0:1784948768.182580:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948768.182584:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784948768.182586:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000010:3.0:1784948768.182588:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 02000000:00000001:2.0:1784948768.182590:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:3.0:1784948768.182596:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000100:00000001:2.0:1784948768.182597:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:0.0:1784948768.182598:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 676 00000100:00000001:2.0:1784948768.182604:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948768.182609:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:2.0:1784948768.182610:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948768.182610:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948768.182617:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470017024 : -115544239534592 : ffff96e9c2a58e00) 00000100:00001000:2.0:1784948768.182623:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=90 v=5 (5 5 5 5) 02000000:00000001:3.0:1784948768.182624:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000040:0.0:1784948768.182629:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000007676ae3f x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:429/0 lens 224/0 e 0 to 0 dl 1784948779 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784948768.182631:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 00000100:00001000:2.0:1784948768.182637:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=90 v=5 (1 1 1 1) 02000000:00000001:3.0:1784948768.182640:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948768.182646:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948768.182653:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000200:3.0:1784948768.182658:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948768.182659:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.182670:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:2.0:1784948768.182671:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:3.0:1784948768.182672:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e686bcb0. 00000020:00000010:0.0:1784948768.182677:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:0.0:1784948768.182683:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784948768.182687:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948768.182690:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000400:00000010:3.0:1784948768.182691:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000105270ec. 00000100:00000001:2.0:1784948768.182693:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948768.182704:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000007676ae3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289651712:12345-192.168.202.137@tcp:400: 00000100:00000040:2.0:1784948768.182707:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948768.182717:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289651712 00000020:00000001:0.0:1784948768.182723:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784948768.182729:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784948768.182731:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:2.0:1784948768.182734:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1784948768.182737:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.182738:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948768.182747:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:2.0:1784948768.182752:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948768.182753:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948768.182763:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948768.182769:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948768.182778:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00100000:2.0:1784948768.182779:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289651456:0@lo:400: 00000020:00000001:0.0:1784948768.182782:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948768.182787:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948768.182791:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948768.182792:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948768.182807:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948768.182808:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948768.182813:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784948768.182815:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000086d6bb31 x1871651289651456/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948768.182819:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b9e025c9. 02000000:00000001:0.0:1784948768.182826:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.182834:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948768.182841:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948768.182848:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289651712 00000100:00000001:2.0:1784948768.182851:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:0.0:1784948768.182854:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:2.0:1784948768.182859:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00010000:00000200:0.0:1784948768.182864:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000007676ae3f x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:429/0 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784948768.182897:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 00010000:00000001:0.0:1784948768.182939:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948768.182943:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948768.182946:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=85 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948768.182951:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948768.182955:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948768.182958:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948768.182961:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948768.182963:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948768.182966:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948768.182970:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:0.0:1784948768.182971:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784948768.182979:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000010:0.0:1784948768.182980:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000020:00000040:2.0:1784948768.182986:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000200:0.0:1784948768.182989:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289651712, offset 224 00000020:00000001:2.0:1784948768.182997:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:0.0:1784948768.183003:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000010:2.0:1784948768.183006:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c03b4f3a. 02000000:00000001:2.0:1784948768.183015:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:0.0:1784948768.183016:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:2.0:1784948768.183021:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:0.0:1784948768.183028:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3212:3212:241:4294967295] 192.168.202.137@tcp LPNI seq info [2086:2086:8:4294967295] 00000100:00000010:2.0:1784948768.183029:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000086d6bb31. 00000100:00000001:2.0:1784948768.183058:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948768.183066:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948768.183073:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784948768.183076:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948768.183086:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948768.183092:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:0.0:1784948768.183093:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000595fa8f8. 00000100:00000001:2.0:1784948768.183122:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948768.183129:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784948768.183132:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:0.0:1784948768.183138:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784948768.183139:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948768.183144:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000595fa8f8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948768.183160:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948768.183164:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948768.183166:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948768.183168:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948768.183170:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948768.183175:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007676ae3f x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:429/0 lens 224/224 e 0 to 0 dl 1784948779 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948768.183195:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000007676ae3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289651712:12345-192.168.202.137@tcp:400: Request processed in 524us (933us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948768.183202:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 676 00000100:00000040:0.0:1784948768.183206:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948768.183208:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948768.183210:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948768.183212:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002632f5ee. 00000020:00000010:0.0:1784948768.183215:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948768.183219:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000069d2b91d. 00000020:00000040:0.0:1784948768.183224:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948768.183228:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948768.183252:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948768.183257:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000595fa8f8. 00000400:00000200:1.0:1784948768.183263:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948768.183270:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948768.183275:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:1.0:1784948768.183277:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000100:00000001:1.0:1784948768.183281:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948768.183284:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948768.183284:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948768.183296:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948768.183300:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948768.183303:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948768.183312:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948768.183322:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435a00 00000400:00000200:2.0:1784948768.183327:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8281 [1] + 224 00000800:00000001:2.0:1784948768.183332:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948768.183335:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948768.183337:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948768.183341:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948768.183345:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948768.183348:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948768.183367:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948768.183384:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948768.183406:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948768.183409:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.183501:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948768.183507:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948768.183514:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948768.183516:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948768.183519:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a0670fc. 00000400:00000200:3.0:1784948768.183527:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:3.0:1784948768.183529:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000100:00000001:3.0:1784948768.183533:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948768.183540:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948768.183552:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948768.183561:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948768.183563:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.183566:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948768.183568:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948768.183570:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948768.183571:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948768.183574:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948768.183579:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=85 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948768.183583:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=85 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948768.183586:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948768.183588:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948768.183590:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948768.183594:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948768.183603:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.183606:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948768.183608:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948768.183610:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948768.183611:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948768.183614:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948768.183622:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948768.183624:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948768.183626:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948768.183629:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948768.183637:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000edab6e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289651712:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948768.183642:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.183662:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948768.183665:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000edab6e1 x1871651289651712/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948768.183672:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948768.183674:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948768.183677:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:3.0:1784948768.183680:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948768.183682:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948768.183683:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948768.183686:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948768.183688:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000762a0fe3. 02000000:00000001:3.0:1784948768.183691:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948768.183693:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948768.183694:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000edab6e1. 00000100:00000001:3.0:1784948768.183698:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948768.183699:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.183702:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948768.183707:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948768.183709:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948768.183711:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948768.183713:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948768.224032:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305707563, at 4305707564 00000100:00000040:0.0:1784948768.224060:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305707563, at 4305707564 00000100:00000010:2.0:1784948768.224064:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000010:0.0:1784948768.224076:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:3.0:1784948768.224096:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305707563, at 4305707564 00000100:00000010:3.0:1784948768.224126:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:1.0:1784948768.224176:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305707563, at 4305707564 00000100:00000010:1.0:1784948768.224188:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00010000:00000001:2.0:1784948769.012031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948769.012041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948769.012050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948769.012054:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948769.012057:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948769.012061:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948769.012065:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948769.012067:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948769.012070:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948769.012073:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948769.012186:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948769.012194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948769.012200:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948770.035989:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948770.035995:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948770.036000:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948770.036001:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948770.036045:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948770.036049:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948770.036052:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948771.060039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948771.060048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948771.060057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948771.060059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948771.060144:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948771.060151:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948771.060155:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948772.084009:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948772.084017:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948772.084024:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948772.084026:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948772.084093:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948772.084099:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948772.084102:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948773.107997:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948773.108004:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948773.108010:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948773.108012:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948773.108099:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948773.108105:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948773.108110:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948773.300054:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948773.300074:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948773.300077:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948773.300085:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948773.300090:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948773.300094:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.300098:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948773.300140:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948773.300155:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948773.300165:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d0fcbe33. 00000020:00000040:0.0:1784948773.300175:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948773.300182:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948773.300187:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948773.300195:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948773.300208:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f8a82730. 00000100:00000001:0.0:1784948773.300220:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948773.300240:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948773.300264:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948773.300269:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948773.300280:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d0fcbe33] to pc [ptlrpcd_00_03+3] req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948773.300325:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948773.300330:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948773.300337:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948773.300342:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948773.300350:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948773.300357:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000075b1f5f6. 00000020:00000040:0.0:1784948773.300364:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948773.300369:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948773.300373:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948773.300378:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948773.300386:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f64a2f6c. 00000100:00000001:0.0:1784948773.300393:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948773.300406:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948773.300417:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948773.300419:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948773.300426:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b1f5f6] to pc [ptlrpcd_00_00+0] req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948773.300443:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784948773.300452:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:2.0:1784948773.300456:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948773.300461:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.300464:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948773.300469:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948773.300472:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11050) 00000100:00000001:2.0:1784948773.300474:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948773.300479:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948773.300482:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948773.300492:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948773.300510:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948773.300514:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948773.300519:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784948773.300521:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784948773.300524:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948773.300528:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948773.300531:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948773.300539:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000d0fcbe33 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289651840:0@lo:400: 00000100:00000001:0.0:1784948773.300540:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948773.300548:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948773.300549:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948773.300553:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1784948773.300559:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784948773.300567:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948773.300568:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948773.300577:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784948773.300586:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 00000100:00000001:0.0:1784948773.300589:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1784948773.300594:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948773.300596:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948773.300604:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:2.0:1784948773.300606:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000096c4b4ae. 02000000:00000001:0.0:1784948773.300607:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948773.300616:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948773.300622:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000075b1f5f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289651968:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948773.300629:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:2.0:1784948773.300634:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 02000000:00000001:0.0:1784948773.300635:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000200:2.0:1784948773.300647:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289651840, portal 25 00000100:00000001:3.0:1784948773.300651:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948773.300653:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.300654:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948773.300659:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558292224 : -115544151259392 : ffff96e9c7e88700) 00000100:00000001:3.0:1784948773.300663:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948773.300665:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948773.300679:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 00000100:00000040:2.0:1784948773.300679:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948773.300692:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.300723:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:0.0:1784948773.300726:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001881934. 00000100:00000040:2.0:1784948773.300728:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948773.300736:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000010:0.0:1784948773.300742:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000200:2.0:1784948773.300743:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289651840, offset 0 00000100:00000200:0.0:1784948773.300758:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289651968, portal 10 00000400:00000200:2.0:1784948773.300759:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948773.300769:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948773.300773:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558301184 : -115544151250432 : ffff96e9c7e8aa00) 00000400:00000200:2.0:1784948773.300782:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:0.0:1784948773.300790:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948773.300800:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435a80 00000400:00000200:2.0:1784948773.300818:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x78ed [2] + 0 00000100:00000001:0.0:1784948773.300821:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:2.0:1784948773.300825:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b4e968f8. 00000400:00000200:2.0:1784948773.300854:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000017500e72 00000100:00000040:0.0:1784948773.300861:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948773.300866:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:0.0:1784948773.300870:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000400:00000200:2.0:1784948773.300875:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:0.0:1784948773.300883:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289651968, offset 0 00000400:00000200:2.0:1784948773.300886:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000017500e72 00000400:00000010:2.0:1784948773.300893:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000017500e72. 00000100:00000001:2.0:1784948773.300898:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948773.300902:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784948773.300939:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000974405de x1871651289651840 msgsize 224 00000400:00000200:0.0:1784948773.300946:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:2.0:1784948773.300949:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948773.300956:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784948773.300966:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948773.300975:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784948773.300977:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948773.300983:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948773.300989:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948773.300993:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948773.300999:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3213:3213:241:4294967295] 192.168.202.137@tcp LPNI seq info [2087:2087:8:4294967295] 00000100:00000001:3.0:1784948773.301009:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948773.301014:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289651840 02000000:00000001:3.0:1784948773.301022:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948773.301027:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:0.0:1784948773.301033:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948773.301035:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948773.301047:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948773.301058:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948773.301061:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000100:00100000:3.0:1784948773.301061:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289651840 00000800:00000010:0.0:1784948773.301069:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a3cf4688. 00000400:00000010:2.0:1784948773.301071:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000020:00000001:3.0:1784948773.301071:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948773.301080:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784948773.301084:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1784948773.301088:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:0.0:1784948773.301094:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000040:3.0:1784948773.301098:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000200:2.0:1784948773.301099:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948773.301111:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:0.0:1784948773.301121:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:3.0:1784948773.301125:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948773.301129:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948773.301137:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948773.301138:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948773.301148:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004562b7bd. 00000800:00000200:0.0:1784948773.301155:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a3cf4688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948773.301160:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948773.301162:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.301172:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784948773.301174:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004562b7bd. 00000100:00000001:2.0:1784948773.301182:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948773.301186:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000100:00000001:0.0:1784948773.301187:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.301190:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305712683 00000020:00000010:3.0:1784948773.301197:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bb54a175. 00000100:00000040:0.0:1784948773.301203:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2042 usecs (3 jiffies) 00000100:00000010:2.0:1784948773.301203:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e19b789f. 00000020:00000010:3.0:1784948773.301206:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:2.0:1784948773.301216:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948773.301218:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305712644 00000100:00000040:3.0:1784948773.301224:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948773.301229:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305712683 00000100:00000001:0.0:1784948773.301230:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305712683 00000100:00000001:3.0:1784948773.301232:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948773.301238:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000010:2.0:1784948773.301239:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000ef8f137f. 00000100:00000001:0.0:1784948773.301242:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305712644 00000100:00000001:3.0:1784948773.301243:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.301246:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948773.301258:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305712683 00000100:00000001:0.0:1784948773.301259:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305712683 00000100:00000001:3.0:1784948773.301260:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948773.301264:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000100:00000001:0.0:1784948773.301274:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305712644 00000100:00000001:2.0:1784948773.301276:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948773.301281:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948773.301285:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305712683 00000100:00000001:0.0:1784948773.301287:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305712683 00000100:00000010:2.0:1784948773.301291:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e2039f57. 00000100:00000001:0.0:1784948773.301296:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305712644 00000100:00000001:2.0:1784948773.301300:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948773.301303:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948773.301309:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948773.301311:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948773.301317:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305712683 00000100:00000001:2.0:1784948773.301318:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948773.301320:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 701 00000100:00000001:2.0:1784948773.301327:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948773.301331:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948773.301335:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948773.301336:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948773.301341:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948773.301347:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180543568 : -115539529008048 : ffff96eadb6a7c50) 00000100:00000001:0.0:1784948773.301351:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948773.301359:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948773.301367:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000974405de x1871651289651840/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:434/0 lens 224/0 e 0 to 0 dl 1784948784 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948773.301370:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948773.301377:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948773.301389:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948773.301395:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948773.301400:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948773.301403:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948773.301411:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948773.301467:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784948773.301480:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004562b7bd. 00000800:00000200:2.0:1784948773.301481:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948773.301482:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948773.301489:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:0.0:1784948773.301492:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784948773.301494:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a3cf4688. 00000020:00000010:3.0:1784948773.301495:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004562b7bd. 00000800:00000001:0.0:1784948773.301500:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948773.301510:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:3.0:1784948773.301511:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000974405de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289651840:12345-0@lo:400: 00000400:00000200:0.0:1784948773.301519:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:3.0:1784948773.301528:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289651840 00000400:00000200:2.0:1784948773.301535:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784948773.301536:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948773.301543:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784948773.301547:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435b00 00000020:00000001:3.0:1784948773.301550:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948773.301551:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000020:00000001:3.0:1784948773.301559:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:2.0:1784948773.301561:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000020:00000001:3.0:1784948773.301564:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784948773.301570:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:2.0:1784948773.301573:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1784948773.301574:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948773.301582:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 152688 00000020:00000001:3.0:1784948773.301584:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948773.301590:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:2.0:1784948773.301591:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948773.301597:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948773.301600:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948773.301608:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948773.301616:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948773.301617:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:0.0:1784948773.301619:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948773.301620:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784948773.301626:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948773.301628:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:3.0:1784948773.301630:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 02000000:00000001:3.0:1784948773.301640:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948773.301649:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948773.301650:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948773.301656:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948773.301658:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.301664:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948773.301665:0:337113:0:(events.c:286:request_in_callback()) Process entered 00010000:00000040:3.0:1784948773.301670:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289651840 00000100:00000200:0.0:1784948773.301672:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00010000:00000001:3.0:1784948773.301677:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000010:0.0:1784948773.301681:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a191ee5f. 00010000:00000200:3.0:1784948773.301688:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000974405de x1871651289651840/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:434/0 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948773.301689:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000a191ee5f x1871651289651968 msgsize 224 00000100:00100000:0.0:1784948773.301714:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:3.0:1784948773.301715:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948773.301720:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948773.301726:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=95 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948773.301733:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948773.301735:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:3.0:1784948773.301757:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948773.301763:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948773.301769:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000800:00000001:0.0:1784948773.301771:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784948773.301773:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948773.301782:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:0.0:1784948773.301783:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948773.301792:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948773.301804:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000200:3.0:1784948773.301810:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289651840, offset 224 00000100:00000001:1.0:1784948773.301816:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:3.0:1784948773.301820:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:1.0:1784948773.301825:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289651968 02000000:00000001:1.0:1784948773.301831:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948773.301858:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948773.301862:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948773.301870:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948773.301888:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:1.0:1784948773.301898:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289651968 00000020:00000001:1.0:1784948773.301902:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948773.301905:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948773.301911:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784948773.301912:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435a80 00000020:00000040:1.0:1784948773.301915:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:3.0:1784948773.301975:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8295 [1] + 224 00000400:00000200:3.0:1784948773.301985:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948773.301992:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:1.0:1784948773.301996:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948773.302000:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1784948773.302001:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:3.0:1784948773.302009:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948773.302010:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948773.302018:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00000001:1.0:1784948773.302028:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948773.302033:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948773.302036:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000020:00000010:1.0:1784948773.302050:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000020:00000010:1.0:1784948773.302056:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ccd24904. 00000020:00000010:1.0:1784948773.302061:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:3.0:1784948773.302063:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1784948773.302074:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000100:00000001:2.0:1784948773.302076:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948773.302077:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:3.0:1784948773.302079:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:2.0:1784948773.302082:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948773.302086:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948773.302090:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948773.302091:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000001:2.0:1784948773.302094:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784948773.302099:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948773.302100:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1784948773.302101:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948773.302108:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000096c4b4ae. 00000100:00000001:1.0:1784948773.302113:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948773.302114:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948773.302116:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000100:00000001:1.0:1784948773.302121:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000010:2.0:1784948773.302124:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000100:00000001:1.0:1784948773.302127:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948773.302131:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784948773.302133:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:3.0:1784948773.302142:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:2.0:1784948773.302144:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948773.302145:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948773.302148:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948773.302161:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784948773.302165:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000200:2.0:1784948773.302170:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948773.302172:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:3.0:1784948773.302177:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000974405de x1871651289651840/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:434/0 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948773.302191:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 677 00000100:00000040:1.0:1784948773.302203:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948773.302207:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948773.302212:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558304768 : -115544151246848 : ffff96e9c7e8b800) 00000100:00000001:2.0:1784948773.302214:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948773.302224:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:1.0:1784948773.302230:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a191ee5f x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:434/0 lens 224/0 e 0 to 0 dl 1784948784 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948773.302233:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948773.302239:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:3.0:1784948773.302243:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000974405de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289651840:12345-0@lo:400: Request processed in 833us (1292us total) trans 0 rc 0/0 00000100:00000001:2.0:1784948773.302244:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948773.302254:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948773.302265:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=95 v=5 (5 5 5 5) 00000100:00000001:1.0:1784948773.302275:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00001000:2.0:1784948773.302277:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=95 v=5 (1 1 1 1) 00000020:00000010:1.0:1784948773.302285:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00100000:3.0:1784948773.302285:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 701 00000100:00000001:2.0:1784948773.302287:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948773.302292:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948773.302295:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784948773.302298:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000040:3.0:1784948773.302299:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000010:1.0:1784948773.302301:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00010000:00000001:2.0:1784948773.302304:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.302312:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948773.302312:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948773.302320:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00100000:1.0:1784948773.302322:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a191ee5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289651968:12345-192.168.202.137@tcp:400: 00000100:00000040:2.0:1784948773.302328:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948773.302333:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bb54a175. 00000100:00000200:1.0:1784948773.302334:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289651968 00000020:00000001:1.0:1784948773.302344:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000010:3.0:1784948773.302348:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000100:00000001:2.0:1784948773.302352:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948773.302353:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000010:3.0:1784948773.302355:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000020:00000001:1.0:1784948773.302360:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.302361:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948773.302366:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784948773.302368:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:1.0:1784948773.302369:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948773.302375:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000040:2.0:1784948773.302376:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948773.302380:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:1.0:1784948773.302386:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 02000000:00000010:3.0:1784948773.302387:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 00000020:00000001:1.0:1784948773.302394:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948773.302398:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:1.0:1784948773.302404:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784948773.302406:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948773.302411:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00100000:2.0:1784948773.302415:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000d0fcbe33 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289651840:0@lo:400: 00000100:00000200:3.0:1784948773.302416:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:1.0:1784948773.302417:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948773.302427:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000010:3.0:1784948773.302429:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002fbcdf42. 00000100:00000001:1.0:1784948773.302435:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948773.302436:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948773.302440:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000049294c87. 02000000:00000001:1.0:1784948773.302442:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948773.302445:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1784948773.302453:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000088553e34. 00000100:00000040:2.0:1784948773.302458:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000d0fcbe33 x1871651289651840/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948773.302460:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948773.302467:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948773.302475:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948773.302482:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289651968 00010000:00000001:1.0:1784948773.302490:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784948773.302496:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000200:1.0:1784948773.302503:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a191ee5f x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:434/0 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948773.302506:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948773.302514:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 00010000:00000001:1.0:1784948773.302518:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948773.302531:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948773.302533:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00001000:1.0:1784948773.302534:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=90 v=5 (1 1 1 1) 00000020:00000001:2.0:1784948773.302540:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1784948773.302542:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:2.0:1784948773.302547:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000040:1.0:1784948773.302549:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948773.302559:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:2.0:1784948773.302560:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:1.0:1784948773.302562:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948773.302568:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948773.302568:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f8a82730. 00000100:00000001:1.0:1784948773.302575:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948773.302579:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1784948773.302581:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948773.302584:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948773.302588:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000d0fcbe33. 00000400:00000010:1.0:1784948773.302589:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000001:2.0:1784948773.302598:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000200:1.0:1784948773.302600:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289651968, offset 224 00000100:00000001:2.0:1784948773.302602:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948773.302610:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948773.302617:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948773.302620:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948773.302624:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948773.302628:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948773.302633:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948773.302633:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948773.302644:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3214:3214:241:4294967295] 192.168.202.137@tcp LPNI seq info [2088:2088:8:4294967295] 00000400:00000200:1.0:1784948773.302660:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948773.302666:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948773.302670:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002a753d9d. 00000800:00000200:1.0:1784948773.302676:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948773.302683:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948773.302686:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002a753d9d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948773.302710:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948773.302714:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948773.302717:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948773.302718:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948773.302721:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948773.302724:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a191ee5f x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:434/0 lens 224/224 e 0 to 0 dl 1784948784 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948773.302737:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a191ee5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289651968:12345-192.168.202.137@tcp:400: Request processed in 461us (1046us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948773.302766:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 677 00000100:00000040:1.0:1784948773.302770:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948773.302773:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948773.302777:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948773.302781:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ccd24904. 00000020:00000010:1.0:1784948773.302786:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948773.302790:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000020:00000040:1.0:1784948773.302795:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948773.302799:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948773.302904:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948773.302947:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002a753d9d. 00000400:00000200:2.0:1784948773.302957:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948773.302968:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948773.302970:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948773.302981:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:2.0:1784948773.302985:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000800:00000001:0.0:1784948773.302988:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948773.302993:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784948773.302997:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948773.303000:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948773.303007:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948773.303022:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948773.303041:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435b00 00000400:00000200:0.0:1784948773.303051:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x829d [1] + 224 00000800:00000001:0.0:1784948773.303060:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948773.303069:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948773.303073:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948773.303081:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948773.303089:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948773.303095:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948773.303114:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948773.303149:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948773.303160:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948773.303183:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948773.303197:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948773.303202:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948773.303208:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948773.303211:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948773.303214:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000001881934. 00000400:00000200:3.0:1784948773.303221:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:3.0:1784948773.303223:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:3.0:1784948773.303227:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948773.303233:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948773.303245:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948773.303254:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948773.303255:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948773.303258:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948773.303260:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948773.303262:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948773.303264:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948773.303266:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948773.303272:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=90 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948773.303276:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=90 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948773.303279:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948773.303280:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948773.303283:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948773.303287:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948773.303295:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948773.303298:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948773.303300:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948773.303303:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948773.303304:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948773.303307:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948773.303315:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948773.303318:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948773.303320:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948773.303323:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948773.303332:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000075b1f5f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289651968:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948773.303337:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948773.303339:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948773.303341:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000075b1f5f6 x1871651289651968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948773.303357:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948773.303359:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948773.303361:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 02000000:00000001:3.0:1784948773.303365:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948773.303367:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948773.303368:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948773.303371:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948773.303373:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f64a2f6c. 02000000:00000001:3.0:1784948773.303376:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948773.303377:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948773.303379:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000075b1f5f6. 00000100:00000001:3.0:1784948773.303383:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948773.303385:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948773.303386:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948773.303391:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948773.303393:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948773.303395:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948773.303397:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948773.344015:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305712683, at 4305712684 00000100:00000010:2.0:1784948773.344033:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00000100:00000040:1.0:1784948773.344043:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305712683, at 4305712684 00000100:00000040:3.0:1784948773.344060:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305712683, at 4305712684 00000100:00000010:1.0:1784948773.344061:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000ef8f137f. 00000100:00000040:0.0:1784948773.344073:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305712683, at 4305712684 00000100:00000010:3.0:1784948773.344080:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00000100:00000010:0.0:1784948773.344094:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e19b789f. 00010000:00000001:2.0:1784948774.132039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948774.132049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948774.132057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948774.132059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948774.132190:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948774.132196:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948774.132200:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948774.324052:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948774.324068:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948774.324070:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948774.324076:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948774.324079:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948774.324083:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948774.324086:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948775.156012:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948775.156019:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948775.156024:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948775.156025:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948775.156079:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948775.156084:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948775.156087:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948775.347987:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948775.348000:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948775.348001:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948775.348006:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948775.348008:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948775.348011:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948775.348014:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948776.180029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948776.180039:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948776.180049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948776.180052:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948776.180132:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948776.180138:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948776.180143:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948776.372012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948776.372029:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948776.372032:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948776.372040:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948776.372045:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948776.372050:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948776.372054:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948777.204048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948777.204057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948777.204066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948777.204068:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948777.204169:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948777.204179:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948777.204184:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948777.395988:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948777.396001:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948777.396002:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948777.396008:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948777.396010:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948777.396014:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948777.396016:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948778.228080:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948778.228094:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948778.228106:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948778.228109:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948778.228253:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948778.228259:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948778.228264:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1784948778.420091:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948778.420102:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948778.420111:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948778.420118:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948778.420123:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784948778.420124:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086d6bb31. 00000100:00000001:1.0:1784948778.420131:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948778.420137:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1784948778.420140:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948778.420143:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948778.420152:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948778.420155:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948778.420163:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948778.420172:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948778.420186:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c03b4f3a. 00000100:00000001:2.0:1784948778.420199:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948778.420217:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948778.420240:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948778.420246:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948778.420257:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086d6bb31] to pc [ptlrpcd_00_01+1] req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948778.420326:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948778.420334:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948778.420343:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948778.420350:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948778.420353:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948778.420360:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948778.420363:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948778.420365:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948778.420368:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:2.0:1784948778.420374:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000078cad088. 00000100:00000001:3.0:1784948778.420377:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948778.420378:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784948778.420386:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948778.420405:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948778.420407:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948778.420412:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948778.420415:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948778.420417:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948778.420420:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948778.420424:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1784948778.420429:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289652096:0@lo:400: 02000000:00000001:2.0:1784948778.420433:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948778.420437:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948778.420449:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1784948778.420449:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000069a945c8. 02000000:00000001:1.0:1784948778.420454:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948778.420461:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948778.420463:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948778.420470:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000229221cc. 02000000:00000001:1.0:1784948778.420476:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948778.420480:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1784948778.420488:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d40cdb47. 00000400:00000010:1.0:1784948778.420504:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000100:00000001:2.0:1784948778.420507:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948778.420513:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948778.420515:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289652096, portal 25 00000100:00000001:1.0:1784948778.420521:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948778.420526:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470019712 : -115544239531904 : ffff96e9c2a59880) 00000100:00000040:2.0:1784948778.420526:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000078cad088] to pc [ptlrpcd_00_02+2] req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948778.420543:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948778.420569:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948778.420576:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948778.420582:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000100:00000200:1.0:1784948778.420587:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289652096, offset 0 00000400:00000200:1.0:1784948778.420594:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948778.420603:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948778.420615:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:2.0:1784948778.420616:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11055) 00000100:00000001:0.0:1784948778.420621:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948778.420637:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435b80 00000100:00000001:0.0:1784948778.420644:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948778.420651:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7911 [2] + 0 00000100:00000001:0.0:1784948778.420652:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:1.0:1784948778.420661:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ccad1f73. 00000100:00000040:0.0:1784948778.420669:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948778.420670:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948778.420671:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000008522cb9f 00000100:00000001:2.0:1784948778.420681:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948778.420687:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948778.420695:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948778.420703:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948778.420704:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784948778.420715:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784948778.420720:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008522cb9f 02000000:00000001:0.0:1784948778.420721:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948778.420728:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000008522cb9f. 02000000:00000001:0.0:1784948778.420729:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948778.420739:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289652224:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948778.420740:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948778.420747:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784948778.420749:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784948778.420756:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000475c8050 x1871651289652096 msgsize 224 02000000:00000001:0.0:1784948778.420757:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948778.420765:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948778.420772:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1784948778.420778:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 02000000:00000010:0.0:1784948778.420783:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 00000100:00000040:1.0:1784948778.420784:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 02000000:00000001:0.0:1784948778.420825:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948778.420834:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000faa31c82. 00000400:00000010:0.0:1784948778.420842:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000553902b5. 00000100:00000200:0.0:1784948778.420848:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289652224, portal 10 00000100:00000001:0.0:1784948778.420852:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948778.420854:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470018816 : -115544239532800 : ffff96e9c2a59500) 00000100:00000040:0.0:1784948778.420865:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948778.420875:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948778.420880:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948778.420901:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000100:00000200:0.0:1784948778.420904:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289652224, offset 0 00000400:00000200:0.0:1784948778.420909:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948778.420944:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948778.420967:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948778.420972:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948778.420977:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948778.420981:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948778.420985:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3215:3215:241:4294967295] 192.168.202.137@tcp LPNI seq info [2089:2089:8:4294967295] 00000400:00000200:0.0:1784948778.421000:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948778.421009:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948778.421016:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d1fbdb0. 00000800:00000200:0.0:1784948778.421026:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948778.421037:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948778.421044:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000000d1fbdb0 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948778.421054:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948778.421064:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948778.421072:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000100:00000040:0.0:1784948778.421074:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2045 usecs (3 jiffies) 00000400:00000010:1.0:1784948778.421080:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000100:00000001:0.0:1784948778.421083:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305717764 00000100:00000001:1.0:1784948778.421091:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:0.0:1784948778.421093:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000200:1.0:1784948778.421102:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948778.421106:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784948778.421136:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948778.421137:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305717764 00000100:00000040:1.0:1784948778.421146:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784948778.421152:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:0.0:1784948778.421166:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948778.421176:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948778.421182:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305717764 00000100:00000001:1.0:1784948778.421182:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948778.421193:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948778.421196:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:3.0:1784948778.421201:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948778.421202:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305717803 00000100:00000001:0.0:1784948778.421209:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00100000:3.0:1784948778.421211:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289652096 00000100:00000001:1.0:1784948778.421218:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305717764 02000000:00000001:3.0:1784948778.421219:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948778.421226:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948778.421228:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305717764 00000100:00000001:3.0:1784948778.421231:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948778.421233:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305717803 02000000:00000001:3.0:1784948778.421244:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948778.421248:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305717764 00000100:00000010:0.0:1784948778.421249:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00100000:3.0:1784948778.421252:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289652096 00000100:00000001:1.0:1784948778.421258:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305717803 00000020:00000001:3.0:1784948778.421262:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948778.421264:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948778.421267:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305717764 00000020:00000040:3.0:1784948778.421269:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948778.421276:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948778.421278:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784948778.421283:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305717803 00000100:00000001:0.0:1784948778.421287:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948778.421289:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948778.421291:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305717764 00000100:00000001:0.0:1784948778.421301:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948778.421302:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948778.421303:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948778.421313:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948778.421316:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948778.421317:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948778.421324:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948778.421328:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948778.421329:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948778.421330:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:3.0:1784948778.421339:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f1666e1d. 00000100:00000001:1.0:1784948778.421340:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948778.421342:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948778.421352:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948778.421353:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948778.421358:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948778.421362:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948778.421362:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f1666e1d. 00000100:00000001:1.0:1784948778.421364:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948778.421377:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948778.421378:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:3.0:1784948778.421379:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e186afc. 00000100:00000001:1.0:1784948778.421386:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948778.421393:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948778.421394:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a39e81d9. 00000100:00000001:1.0:1784948778.421398:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948778.421403:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000040:3.0:1784948778.421421:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948778.421426:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948778.421429:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948778.421432:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948778.421445:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948778.421448:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948778.421451:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948778.421461:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d1fbdb0. 00000100:00000001:3.0:1784948778.421462:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948778.421465:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948778.421474:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948778.421478:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948778.421483:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948778.421490:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784948778.421496:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948778.421497:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784948778.421504:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948778.421511:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000100:00100000:3.0:1784948778.421512:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 702 00000400:00000010:2.0:1784948778.421521:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000040:3.0:1784948778.421522:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:1.0:1784948778.421523:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435c00 00000100:00000001:3.0:1784948778.421533:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180529232 : -115539529022384 : ffff96eadb6a4450) 00000100:00000001:2.0:1784948778.421537:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1784948778.421545:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000475c8050 x1871651289652096/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:439/0 lens 224/0 e 0 to 0 dl 1784948789 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948778.421547:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 152912 00000100:00000200:2.0:1784948778.421553:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948778.421567:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948778.421577:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f1666e1d. 00000800:00000001:1.0:1784948778.421578:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948778.421590:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948778.421591:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784948778.421593:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948778.421599:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f1666e1d. 00000100:00000040:2.0:1784948778.421602:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948778.421603:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:3.0:1784948778.421614:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000475c8050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289652096:12345-0@lo:400: 00000400:00000200:1.0:1784948778.421624:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:3.0:1784948778.421634:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289652096 00000100:00000001:1.0:1784948778.421642:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948778.421648:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:3.0:1784948778.421648:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784948778.421650:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948778.421663:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a07b5f33. 00000100:00000001:2.0:1784948778.421666:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784948778.421669:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:1.0:1784948778.421670:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000a07b5f33 x1871651289652224 msgsize 224 00000020:00000001:3.0:1784948778.421679:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948778.421688:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:3.0:1784948778.421692:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948778.421699:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784948778.421700:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:3.0:1784948778.421711:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000001:1.0:1784948778.421713:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948778.421715:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948778.421720:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948778.421725:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948778.421732:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948778.421738:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948778.421745:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948778.421750:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948778.421757:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948778.421762:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000010:3.0:1784948778.421766:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000077ad1613. 00000100:00100000:2.0:1784948778.421771:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289652224 02000000:00000001:3.0:1784948778.421775:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948778.421780:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948778.421783:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948778.421787:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948778.421815:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948778.421817:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948778.421825:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948778.421827:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289652096 00000100:00100000:2.0:1784948778.421835:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289652224 00010000:00000001:3.0:1784948778.421837:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:2.0:1784948778.421846:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948778.421853:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000200:3.0:1784948778.421854:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000475c8050 x1871651289652096/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:439/0 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948778.421862:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948778.421870:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000001:3.0:1784948778.421882:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000001:2.0:1784948778.421883:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:3.0:1784948778.421891:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948778.421894:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00001000:3.0:1784948778.421902:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=100 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948778.421961:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948778.421963:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1784948778.421974:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000329d026c. 00000100:00000040:3.0:1784948778.421974:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948778.421986:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:2.0:1784948778.421987:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948778.421995:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000010:2.0:1784948778.421999:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000329d026c. 02000000:00000001:3.0:1784948778.422003:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948778.422014:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000100:00000001:3.0:1784948778.422015:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:2.0:1784948778.422026:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002fe80d84. 00000100:00000040:3.0:1784948778.422026:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948778.422036:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000020:00000010:2.0:1784948778.422038:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000200:3.0:1784948778.422047:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289652096, offset 224 00000100:00000040:2.0:1784948778.422069:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:3.0:1784948778.422076:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948778.422080:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948778.422086:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948778.422091:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948778.422099:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784948778.422105:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948778.422112:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784948778.422120:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948778.422120:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435b80 00000400:00000200:3.0:1784948778.422144:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x82b1 [1] + 224 00000100:00000001:2.0:1784948778.422149:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948778.422158:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948778.422171:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948778.422174:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948778.422180:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784948778.422181:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:3.0:1784948778.422196:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948778.422202:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 678 00000100:00000040:2.0:1784948778.422215:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948778.422225:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451655936 : -115544257895680 : ffff96e9c18d6300) 00000100:00000040:3.0:1784948778.422234:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948778.422242:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a07b5f33 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:439/0 lens 224/0 e 0 to 0 dl 1784948789 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948778.422287:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948778.422293:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000329d026c. 00000100:00000001:2.0:1784948778.422300:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948778.422300:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784948778.422310:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000329d026c. 00000400:00000200:3.0:1784948778.422318:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000100:00100000:2.0:1784948778.422324:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a07b5f33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289652224:12345-192.168.202.137@tcp:400: 00000400:00000010:3.0:1784948778.422327:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:3.0:1784948778.422339:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000200:2.0:1784948778.422340:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289652224 00000100:00000001:3.0:1784948778.422345:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:2.0:1784948778.422349:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784948778.422357:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948778.422358:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948778.422366:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948778.422370:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:2.0:1784948778.422375:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00010000:00000001:3.0:1784948778.422380:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948778.422382:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784948778.422387:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948778.422392:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948778.422396:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:2.0:1784948778.422400:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948778.422408:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948778.422409:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:3.0:1784948778.422410:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000475c8050 x1871651289652096/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:439/0 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948778.422417:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948778.422419:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000001:1.0:1784948778.422429:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1784948778.422429:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948778.422436:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1784948778.422441:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000010:1.0:1784948778.422443:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d40cdb47. 00000100:00000001:2.0:1784948778.422453:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:1.0:1784948778.422456:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 02000000:00000001:2.0:1784948778.422461:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000010:1.0:1784948778.422465:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000100:00100000:3.0:1784948778.422466:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000475c8050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289652096:12345-0@lo:400: Request processed in 889us (1700us total) trans 0 rc 0/0 02000000:00000010:2.0:1784948778.422472:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000901ee2b9. 00000100:00000001:1.0:1784948778.422474:0:337120:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1784948778.422483:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948778.422486:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948778.422493:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948778.422503:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 702 00000020:00000001:2.0:1784948778.422506:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948778.422518:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00010000:00000040:2.0:1784948778.422520:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289652224 00000100:00000200:1.0:1784948778.422525:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948778.422530:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00010000:00000001:2.0:1784948778.422535:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948778.422539:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948778.422553:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a39e81d9. 00000100:00000001:1.0:1784948778.422554:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000200:2.0:1784948778.422554:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a07b5f33 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:439/0 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948778.422564:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948778.422574:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:3.0:1784948778.422578:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 02000000:00000001:1.0:1784948778.422580:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948778.422587:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:3.0:1784948778.422593:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005e186afc. 00000100:00000001:1.0:1784948778.422594:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948778.422602:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:1.0:1784948778.422604:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948778.422611:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948778.422612:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:1.0:1784948778.422613:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=100 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948778.422626:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=95 v=5 (1 1 1 1) 00000100:00001000:1.0:1784948778.422628:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=100 v=5 (1 1 1 1) 02000000:00000001:3.0:1784948778.422629:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948778.422637:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000077ad1613. 00000100:00000001:1.0:1784948778.422647:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948778.422648:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784948778.422659:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948778.422660:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000040:2.0:1784948778.422669:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948778.422670:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784948778.422671:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948778.422677:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948778.422682:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:1.0:1784948778.422684:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948778.422687:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:2.0:1784948778.422696:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:1.0:1784948778.422701:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948778.422702:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 02000000:00000001:2.0:1784948778.422704:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948778.422719:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a1933a2b. 00000100:00000001:2.0:1784948778.422720:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948778.422724:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948778.422731:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1784948778.422735:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948778.422737:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948778.422747:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948778.422749:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000100:00100000:1.0:1784948778.422766:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289652096:0@lo:400: 00000100:00000200:2.0:1784948778.422766:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289652224, offset 224 00000100:00000001:1.0:1784948778.422782:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948778.422786:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948778.422820:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784948778.422830:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000086d6bb31 x1871651289652096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948778.422843:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948778.422869:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3216:3216:241:4294967295] 192.168.202.137@tcp LPNI seq info [2090:2090:8:4294967295] 00000100:00000001:1.0:1784948778.422870:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948778.422878:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948778.422885:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000229221cc. 02000000:00000001:1.0:1784948778.422899:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1784948778.422943:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948778.422956:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784948778.422965:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000800:00000010:2.0:1784948778.422966:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d5a127c9. 00000020:00000040:1.0:1784948778.422972:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000800:00000200:2.0:1784948778.422985:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:1.0:1784948778.422986:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948778.422994:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c03b4f3a. 00000800:00000200:2.0:1784948778.423004:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000001:1.0:1784948778.423005:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948778.423010:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:2.0:1784948778.423014:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d5a127c9 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:1.0:1784948778.423015:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000086d6bb31. 00000100:00000001:1.0:1784948778.423024:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948778.423027:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948778.423030:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948778.423037:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948778.423039:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948778.423041:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948778.423045:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948778.423046:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948778.423057:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948778.423063:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948778.423068:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948778.423073:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948778.423082:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a07b5f33 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:439/0 lens 224/224 e 0 to 0 dl 1784948789 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948778.423113:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a07b5f33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289652224:12345-192.168.202.137@tcp:400: Request processed in 823us (1439us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948778.423129:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 678 00000100:00000040:2.0:1784948778.423134:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948778.423140:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948778.423143:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948778.423149:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002fe80d84. 00000020:00000010:2.0:1784948778.423158:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948778.423165:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000020:00000040:2.0:1784948778.423172:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948778.423177:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948778.423235:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948778.423242:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d5a127c9. 00000400:00000200:0.0:1784948778.423248:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948778.423256:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948778.423257:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948778.423268:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000400:00000010:0.0:1784948778.423271:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000800:00000001:1.0:1784948778.423274:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948778.423277:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784948778.423280:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948778.423283:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784948778.423287:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948778.423300:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948778.423317:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435c00 00000400:00000200:1.0:1784948778.423325:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x82b9 [1] + 224 00000800:00000001:1.0:1784948778.423332:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948778.423342:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948778.423345:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948778.423350:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948778.423355:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948778.423359:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948778.423370:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948778.423394:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948778.423401:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948778.423404:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948778.423457:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948778.423464:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948778.423470:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948778.423472:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948778.423475:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000faa31c82. 00000400:00000200:0.0:1784948778.423481:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553902b5 00000400:00000010:0.0:1784948778.423483:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000553902b5. 00000100:00000001:0.0:1784948778.423486:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948778.423492:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948778.423505:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948778.423515:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948778.423517:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948778.423519:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948778.423522:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948778.423524:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948778.423526:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948778.423529:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948778.423535:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=95 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948778.423539:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=95 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948778.423542:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948778.423544:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948778.423547:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948778.423550:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948778.423559:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948778.423562:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948778.423564:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948778.423566:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948778.423568:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948778.423571:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948778.423580:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948778.423582:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948778.423584:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948778.423587:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948778.423596:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289652224:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948778.423601:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948778.423604:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948778.423606:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000078cad088 x1871651289652224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948778.423622:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948778.423624:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948778.423626:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948778.423630:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948778.423632:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948778.423634:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948778.423636:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948778.423638:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000069a945c8. 02000000:00000001:0.0:1784948778.423641:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948778.423643:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948778.423644:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000078cad088. 00000100:00000001:0.0:1784948778.423648:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948778.423649:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948778.423651:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948778.423656:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948778.423658:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948778.423660:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948778.423663:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948778.463988:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305717803, at 4305717804 00000100:00000010:0.0:1784948778.464003:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:2.0:1784948778.464010:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305717803, at 4305717804 00000100:00000040:3.0:1784948778.464017:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305717803, at 4305717804 00000100:00000010:2.0:1784948778.464025:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:3.0:1784948778.464032:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:1.0:1784948778.464044:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305717803, at 4305717804 00000100:00000010:1.0:1784948778.464059:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00010000:00000001:2.0:1784948779.252026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948779.252038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948779.252049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948779.252054:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948779.252058:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948779.252063:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948779.252069:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948779.252073:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948779.252076:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948779.252081:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948779.252202:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948779.252212:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948779.252217:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948779.444258:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948779.444276:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948779.444279:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948779.444285:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948779.444289:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948779.444293:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948779.444296:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948780.275993:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948780.276002:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948780.276009:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948780.276012:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948780.276118:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948780.276123:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948780.276125:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948780.467967:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948780.467979:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948780.467981:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948780.467985:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948780.467989:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948780.467992:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948780.467994:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948781.300042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948781.300052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948781.300060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948781.300061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948781.300126:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948781.300132:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948781.300136:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948781.492017:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948781.492037:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948781.492041:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948781.492049:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948781.492053:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948781.492058:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948781.492063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948782.324066:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948782.324076:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948782.324086:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948782.324088:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948782.324193:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948782.324202:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948782.324209:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948782.515978:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948782.515994:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948782.515996:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948782.516002:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948782.516005:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948782.516008:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948782.516011:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948783.347990:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948783.347996:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948783.348001:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948783.348003:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948783.348086:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948783.348091:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948783.348095:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948783.539987:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948783.540001:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948783.540003:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948783.540008:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948783.540011:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948783.540014:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948783.540015:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948783.540021:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948783.540024:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948783.540036:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b93d109. 00000020:00000040:1.0:1784948783.540043:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948783.540049:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948783.540052:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948783.540059:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948783.540068:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000034aacccc. 00000100:00000001:1.0:1784948783.540075:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948783.540086:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948783.540096:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948783.540098:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948783.540102:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005b93d109] to pc [ptlrpcd_00_03+3] req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948783.540127:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948783.540129:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:0.0:1784948783.540133:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1784948783.540133:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1784948783.540139:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948783.540139:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948783.540144:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948783.540145:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948783.540150:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b47dc450. 00000100:00000040:0.0:1784948783.540153:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784948783.540154:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948783.540159:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948783.540159:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948783.540162:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948783.540164:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948783.540166:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948783.540174:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784948783.540175:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948783.540177:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540179:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948783.540184:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784948783.540184:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1784948783.540189:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948783.540195:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948783.540196:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948783.540199:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948783.540202:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289652352:0@lo:400: 00000100:00000040:1.0:1784948783.540205:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b47dc450] to pc [ptlrpcd_00_00+0] req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948783.540212:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948783.540219:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948783.540223:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948783.540227:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948783.540228:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948783.540235:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 00000100:00000040:1.0:1784948783.540235:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11060) 02000000:00000001:0.0:1784948783.540241:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948783.540243:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948783.540246:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:0.0:1784948783.540247:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000faa31c82. 00000100:00000001:3.0:1784948783.540248:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948783.540251:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948783.540254:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948783.540258:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000553902b5. 00000100:00000040:3.0:1784948783.540259:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948783.540268:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289652352, portal 25 00000100:00000001:0.0:1784948783.540274:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948783.540276:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948783.540277:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451647872 : -115544257903744 : ffff96e9c18d4380) 00000100:00000001:3.0:1784948783.540280:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948783.540286:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948783.540287:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948783.540291:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948783.540298:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540305:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1784948783.540307:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b47dc450 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289652480:192.168.202.137@tcp:400: 00000100:00000040:0.0:1784948783.540312:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948783.540318:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000100:00000001:3.0:1784948783.540318:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948783.540322:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000200:0.0:1784948783.540324:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289652352, offset 0 02000000:00000001:3.0:1784948783.540326:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948783.540330:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948783.540334:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:3.0:1784948783.540335:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a7af9ab9. 02000000:00000001:3.0:1784948783.540340:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948783.540343:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000400:00000010:3.0:1784948783.540347:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000400:00000200:0.0:1784948783.540352:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:3.0:1784948783.540353:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289652480, portal 10 00000100:00000001:3.0:1784948783.540357:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948783.540358:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451656832 : -115544257894784 : ffff96e9c18d6680) 00000400:00000200:0.0:1784948783.540364:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435c80 00000100:00000040:3.0:1784948783.540365:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948783.540375:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x792d [2] + 0 00000100:00000001:3.0:1784948783.540377:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:0.0:1784948783.540380:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ecb3ce7f. 00000100:00000040:3.0:1784948783.540385:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948783.540386:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000000a17d4a2 00000400:00000010:3.0:1784948783.540388:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000100:00000200:3.0:1784948783.540391:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289652480, offset 0 00000400:00000200:0.0:1784948783.540395:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948783.540396:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948783.540402:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948783.540408:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948783.540412:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948783.540416:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948783.540419:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948783.540421:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948783.540424:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3217:3217:241:4294967295] 192.168.202.137@tcp LPNI seq info [2091:2091:8:4294967295] 00000400:00000200:3.0:1784948783.540435:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948783.540439:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948783.540440:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000a17d4a2 00000400:00000010:0.0:1784948783.540444:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000000a17d4a2. 00000800:00000010:3.0:1784948783.540444:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784948783.540453:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948783.540456:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948783.540458:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:3.0:1784948783.540463:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:0.0:1784948783.540465:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000a73b72ec x1871651289652352 msgsize 224 00000800:00000200:3.0:1784948783.540468:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00100000:0.0:1784948783.540474:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948783.540478:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784948783.540487:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540492:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:3.0:1784948783.540493:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2047 usecs (3 jiffies) 00000100:00000001:3.0:1784948783.540498:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305722883 00000100:00000010:3.0:1784948783.540502:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000400:00000200:0.0:1784948783.540504:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:0.0:1784948783.540508:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000100:00000001:3.0:1784948783.540509:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948783.540514:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948783.540516:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305722883 00000100:00000200:0.0:1784948783.540520:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948783.540521:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:3.0:1784948783.540526:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948783.540531:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305722883 00000100:00000010:3.0:1784948783.540536:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:3.0:1784948783.540541:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948783.540550:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948783.540551:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305722883 00000100:00000040:0.0:1784948783.540555:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948783.540557:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:3.0:1784948783.540563:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948783.540565:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948783.540567:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948783.540568:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948783.540572:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948783.540573:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948783.540576:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540577:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948783.540578:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289652352 00000100:00000001:3.0:1784948783.540579:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948783.540584:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948783.540585:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305722922 00000100:00000001:2.0:1784948783.540587:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948783.540588:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948783.540591:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540592:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305722883 00000100:00000001:3.0:1784948783.540592:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948783.540599:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948783.540601:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305722922 02000000:00000001:2.0:1784948783.540601:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948783.540601:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948783.540605:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540608:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305722883 00000100:00100000:2.0:1784948783.540609:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289652352 00000100:00000001:3.0:1784948783.540609:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540616:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305722922 00000020:00000001:2.0:1784948783.540616:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948783.540619:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948783.540622:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305722883 00000020:00000001:2.0:1784948783.540623:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948783.540627:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305722922 00000020:00000040:2.0:1784948783.540627:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948783.540632:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305722883 00000020:00000001:2.0:1784948783.540632:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784948783.540635:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948783.540637:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948783.540640:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948783.540641:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948783.540646:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948783.540647:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000046d06efb. 00000100:00000001:0.0:1784948783.540648:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948783.540650:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948783.540653:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1784948783.540654:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000001:0.0:1784948783.540655:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784948783.540656:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000046d06efb. 00000100:00000001:0.0:1784948783.540659:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:1.0:1784948783.540662:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948783.540663:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1784948783.540664:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000100:00000001:0.0:1784948783.540666:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948783.540671:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948783.540672:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:2.0:1784948783.540673:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002fe80d84. 00000100:00000001:0.0:1784948783.540676:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948783.540680:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000400:00000200:1.0:1784948783.540681:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:1.0:1784948783.540685:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000100:00000001:1.0:1784948783.540690:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1784948783.540693:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000200:1.0:1784948783.540696:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948783.540697:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948783.540701:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000001:0.0:1784948783.540704:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948783.540704:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948783.540710:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948783.540713:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948783.540714:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948783.540716:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948783.540722:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948783.540725:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948783.540727:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948783.540728:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784948783.540729:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948783.540738:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948783.540744:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784948783.540762:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435d00 00000100:00000001:2.0:1784948783.540762:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948783.540771:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 703 00000400:00000200:0.0:1784948783.540773:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 153136 00000100:00000040:2.0:1784948783.540777:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000800:00000001:0.0:1784948783.540780:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948783.540780:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180532304 : -115539529019312 : ffff96eadb6a5050) 00000100:00000040:2.0:1784948783.540786:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a73b72ec x1871651289652352/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:444/0 lens 224/0 e 0 to 0 dl 1784948794 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948783.540788:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948783.540794:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948783.540803:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948783.540810:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948783.540811:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000010:2.0:1784948783.540813:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000046d06efb. 00000100:00000200:0.0:1784948783.540814:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784948783.540818:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1784948783.540820:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000097a95a31. 00000020:00000010:2.0:1784948783.540821:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000046d06efb. 00000100:00000040:0.0:1784948783.540825:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000097a95a31 x1871651289652480 msgsize 224 00000100:00100000:2.0:1784948783.540830:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a73b72ec pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289652352:12345-0@lo:400: 00000100:00100000:0.0:1784948783.540838:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:2.0:1784948783.540839:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289652352 00000020:00000001:2.0:1784948783.540843:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948783.540847:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948783.540848:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948783.540850:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784948783.540851:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:2.0:1784948783.540852:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948783.540856:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948783.540857:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948783.540858:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784948783.540859:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:0.0:1784948783.540862:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948783.540863:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948783.540866:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948783.540869:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948783.540872:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948783.540875:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948783.540880:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006926284c. 02000000:00000001:2.0:1784948783.540882:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948783.540884:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948783.540886:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948783.540888:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289652352 00010000:00000001:2.0:1784948783.540890:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948783.540892:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000200:2.0:1784948783.540895:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a73b72ec x1871651289652352/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:444/0 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948783.540896:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289652480 02000000:00000001:0.0:1784948783.540900:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948783.540928:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00010000:00000001:2.0:1784948783.540928:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948783.540930:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948783.540932:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948783.540935:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948783.540936:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=105 v=5 (1 1 1 1) 00000100:00100000:0.0:1784948783.540939:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289652480 00000020:00000001:0.0:1784948783.540942:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784948783.540942:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:0.0:1784948783.540946:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000040:2.0:1784948783.540948:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000001:0.0:1784948783.540949:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948783.540953:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000040:0.0:1784948783.540954:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:2.0:1784948783.540957:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:0.0:1784948783.540958:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:2.0:1784948783.540959:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948783.540962:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948783.540963:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948783.540965:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784948783.540969:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:0.0:1784948783.540971:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000400:00000010:2.0:1784948783.540973:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000100:00000001:0.0:1784948783.540975:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784948783.540977:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289652352, offset 224 00000020:00000010:0.0:1784948783.540978:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000020:00000010:0.0:1784948783.540985:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000400:00000200:2.0:1784948783.540986:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:0.0:1784948783.540991:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000030e14bca. 00000400:00000200:2.0:1784948783.540995:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784948783.540996:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948783.541004:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:2.0:1784948783.541005:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435c80 00000100:00000001:0.0:1784948783.541008:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948783.541011:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:2.0:1784948783.541012:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x82cd [1] + 224 00000100:00000001:0.0:1784948783.541013:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948783.541019:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:2.0:1784948783.541019:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948783.541023:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000200:2.0:1784948783.541026:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948783.541027:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948783.541031:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948783.541034:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948783.541036:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948783.541046:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948783.541048:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948783.541051:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948783.541058:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 679 00000100:00000040:0.0:1784948783.541061:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948783.541062:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558302080 : -115544151249536 : ffff96e9c7e8ad80) 00000100:00000040:0.0:1784948783.541066:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000097a95a31 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:444/0 lens 224/0 e 0 to 0 dl 1784948794 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948783.541067:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784948783.541074:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000100:00000001:0.0:1784948783.541075:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:2.0:1784948783.541078:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000020:00000010:0.0:1784948783.541079:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000100:00000001:2.0:1784948783.541081:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948783.541083:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948783.541084:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000010:0.0:1784948783.541086:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000100:00000001:2.0:1784948783.541089:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948783.541093:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000097a95a31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289652480:12345-192.168.202.137@tcp:400: 00000100:00000040:2.0:1784948783.541095:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784948783.541099:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1784948783.541100:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:0.0:1784948783.541101:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289652480 00000100:00000001:3.0:1784948783.541102:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948783.541104:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948783.541105:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948783.541107:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:0.0:1784948783.541109:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000001:3.0:1784948783.541110:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1784948783.541111:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948783.541112:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a73b72ec x1871651289652352/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:444/0 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784948783.541113:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1784948783.541116:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:3.0:1784948783.541117:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000faa31c82. 00000020:00000001:0.0:1784948783.541120:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:3.0:1784948783.541124:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553902b5 00000020:00000001:0.0:1784948783.541126:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:3.0:1784948783.541129:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000553902b5. 00000020:00000001:0.0:1784948783.541130:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948783.541135:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00100000:2.0:1784948783.541136:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a73b72ec pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289652352:12345-0@lo:400: Request processed in 324us (668us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948783.541136:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1784948783.541137:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948783.541141:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948783.541144:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948783.541146:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:2.0:1784948783.541150:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 703 00000100:00000001:0.0:1784948783.541152:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948783.541155:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784948783.541158:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 02000000:00000010:0.0:1784948783.541160:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006a5b27c1. 00000100:00000001:2.0:1784948783.541164:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:0.0:1784948783.541165:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948783.541167:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948783.541169:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948783.541169:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:0.0:1784948783.541175:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948783.541175:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002fe80d84. 00010000:00000040:0.0:1784948783.541181:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289652480 00000020:00000010:2.0:1784948783.541183:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000100:00000001:3.0:1784948783.541183:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000001:0.0:1784948783.541184:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948783.541186:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948783.541190:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000100:00000001:3.0:1784948783.541190:0:337122:0:(client.c:1580:after_reply()) Process entered 00010000:00000200:0.0:1784948783.541191:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000097a95a31 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:444/0 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948783.541194:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:2.0:1784948783.541197:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948783.541198:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948783.541202:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948783.541204:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:3.0:1784948783.541208:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948783.541209:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000010:2.0:1784948783.541209:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006926284c. 00010000:00000001:0.0:1784948783.541213:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948783.541215:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:0.0:1784948783.541216:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=100 v=5 (1 1 1 1) 00000100:00001000:3.0:1784948783.541216:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=105 v=5 (5 5 5 5) 00000100:00000001:2.0:1784948783.541218:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948783.541226:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1784948783.541226:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:3.0:1784948783.541227:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=105 v=5 (1 1 1 1) 00000100:00000040:0.0:1784948783.541230:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000400:00000010:2.0:1784948783.541234:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000096c4b4ae. 00000100:00000001:0.0:1784948783.541235:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784948783.541237:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000001:0.0:1784948783.541240:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:2.0:1784948783.541240:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000059548475. 00000100:00000001:3.0:1784948783.541240:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948783.541243:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948783.541247:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784948783.541248:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:3.0:1784948783.541252:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948783.541253:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948783.541259:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000100:00000001:3.0:1784948783.541259:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948783.541262:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289652480, offset 224 00000400:00000200:0.0:1784948783.541268:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784948783.541268:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948783.541277:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948783.541288:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3218:3218:241:4294967295] 192.168.202.137@tcp LPNI seq info [2092:2092:8:4294967295] 00000100:00000001:3.0:1784948783.541292:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948783.541299:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:0.0:1784948783.541301:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948783.541303:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948783.541308:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948783.541313:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d1fbdb0. 00000100:00000040:3.0:1784948783.541313:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948783.541321:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948783.541331:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948783.541335:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000000d1fbdb0 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:3.0:1784948783.541341:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289652352:0@lo:400: 00000100:00000001:0.0:1784948783.541347:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948783.541348:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948783.541353:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948783.541354:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000040:3.0:1784948783.541357:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005b93d109 x1871651289652352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948783.541359:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948783.541362:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948783.541367:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784948783.541369:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948783.541372:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:0.0:1784948783.541374:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000097a95a31 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:444/0 lens 224/224 e 0 to 0 dl 1784948794 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784948783.541375:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 02000000:00000001:3.0:1784948783.541382:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948783.541385:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000100:00100000:0.0:1784948783.541388:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000097a95a31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289652480:12345-192.168.202.137@tcp:400: Request processed in 310us (558us total) trans 0 rc 0/0 00000020:00000040:3.0:1784948783.541389:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784948783.541394:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948783.541397:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000034aacccc. 00000100:00100000:0.0:1784948783.541401:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 679 02000000:00000001:3.0:1784948783.541403:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:0.0:1784948783.541404:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 02000000:00000001:3.0:1784948783.541406:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1784948783.541407:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000010:3.0:1784948783.541408:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b93d109. 00000100:00000001:0.0:1784948783.541411:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948783.541413:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000010:0.0:1784948783.541415:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000030e14bca. 00000100:00000001:3.0:1784948783.541416:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948783.541419:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948783.541420:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784948783.541423:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784948783.541424:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 00000100:00000001:3.0:1784948783.541426:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948783.541429:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948783.541431:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948783.541433:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948783.541434:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948783.541435:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784948783.541439:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d1fbdb0. 00000400:00000200:1.0:1784948783.541443:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948783.541448:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948783.541448:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948783.541454:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:1.0:1784948783.541456:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000100:00000001:1.0:1784948783.541459:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784948783.541459:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948783.541461:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948783.541464:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948783.541470:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948783.541478:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948783.541487:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435d00 00000400:00000200:2.0:1784948783.541492:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x82d5 [1] + 224 00000800:00000001:2.0:1784948783.541497:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948783.541504:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948783.541506:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948783.541510:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948783.541514:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948783.541516:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948783.541522:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948783.541534:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948783.541538:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948783.541540:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948783.541565:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948783.541568:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948783.541572:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948783.541573:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948783.541575:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000400:00000200:3.0:1784948783.541580:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:3.0:1784948783.541582:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:3.0:1784948783.541586:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948783.541591:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948783.541599:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948783.541607:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948783.541608:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948783.541610:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948783.541611:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948783.541613:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948783.541614:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948783.541616:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948783.541621:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=100 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948783.541625:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=100 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948783.541628:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948783.541629:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948783.541631:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948783.541635:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948783.541643:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948783.541645:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948783.541647:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948783.541648:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948783.541649:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948783.541652:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948783.541661:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948783.541663:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948783.541664:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948783.541666:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948783.541671:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b47dc450 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289652480:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948783.541674:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948783.541675:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948783.541676:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b47dc450 x1871651289652480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948783.541681:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948783.541682:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948783.541683:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a7af9ab9. 02000000:00000001:3.0:1784948783.541687:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948783.541689:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948783.541690:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948783.541692:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948783.541693:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:3.0:1784948783.541695:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948783.541696:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948783.541696:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b47dc450. 00000100:00000001:3.0:1784948783.541699:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948783.541699:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948783.541700:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948783.541703:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948783.541704:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948783.541705:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948783.541706:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948783.582958:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305722922, at 4305722923 00000100:00000040:1.0:1784948783.582970:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305722922, at 4305722923 00000100:00000010:0.0:1784948783.582971:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:3.0:1784948783.582973:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305722922, at 4305722923 00000100:00000040:2.0:1784948783.582974:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305722922, at 4305722923 00000100:00000010:1.0:1784948783.582985:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:3.0:1784948783.582986:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00000100:00000010:2.0:1784948783.582989:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784948784.372050:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948784.372060:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948784.372079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948784.372081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948784.372188:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948784.372194:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948784.372199:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948784.563971:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948784.563983:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948784.563985:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948784.563989:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948784.563991:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948784.563994:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948784.563995:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784948784.755983:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.8@tcp 00000800:00000010:2.0:1784948784.755988:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000089a028d. 00000400:00000200:2.0:1784948784.755996:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948784.756001:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948784.756006:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11057 health :900 00000400:00000200:2.0:1784948784.756010:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948784.756013:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:2.0:1784948784.756016:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000800:00000100:2.0:1784948784.756037:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000800:00000010:2.0:1784948784.756038:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e5f52141. 00000400:00000200:2.0:1784948784.756044:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948784.756047:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948784.756050:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948784.756052:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:3.0:1784948784.756053:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948784.756055:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:3.0:1784948784.756065:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:3.0:1784948784.756088:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948784.756091:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948784.756095:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:3.0:1784948784.756098:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000400:00000200:3.0:1784948784.756105:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948784.756111:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:3.0:1784948784.756117:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003f3c01c8. 00000400:00000200:3.0:1784948784.756135:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-110 00000400:00000200:3.0:1784948784.756138:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -110 00000400:00000200:3.0:1784948784.756143:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000400:00000200:3.0:1784948784.756148:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000001:3.0:1784948784.756151:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948784.756153:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948784.756155:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000400:00000010:3.0:1784948784.756157:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000400:00000200:3.0:1784948784.756160:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948784.756163:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:3.0:1784948784.756167:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000061851253. 00000400:00000200:3.0:1784948784.756172:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:3.0:1784948784.756175:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:3.0:1784948784.756177:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00010000:00000001:2.0:1784948785.396015:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948785.396025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948785.396032:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948785.396033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948785.396139:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948785.396145:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948785.396149:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948785.588001:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948785.588016:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948785.588019:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948785.588023:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948785.588026:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948785.588030:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948785.588033:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948785.780080:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948785.780092:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000db05a3db. 00000400:00000010:1.0:1784948785.780106:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:1.0:1784948785.780142:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000010:1.0:1784948785.780170:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000094f3bbd4. 00000400:00000200:1.0:1784948785.780176:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948785.780198:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948785.780212:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948785.780222:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3219:3219:243:4294967295] 192.168.202.137@tcp LPNI seq info [2093:2093:8:4294967295] 00000400:00000200:1.0:1784948785.780234:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948785.780246:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948785.780261:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948785.780271:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000662152ce. 00000800:00000200:1.0:1784948785.780284:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948785.780299:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948785.780306:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000662152ce type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948785.780342:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11059 health :800 00000800:00000200:0.0:1784948785.780557:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948785.780568:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000662152ce. 00000400:00000200:0.0:1784948785.780581:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948785.780610:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784948785.780617:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948785.780619:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948785.780627:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948785.780638:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948785.780646:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948785.780656:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948785.780669:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948785.780705:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948785.780720:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948785.780795:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948785.780803:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3220:3220:243:4294967295] 192.168.202.137@tcp LPNI seq info [2094:2094:8:4294967295] 00000400:00000200:1.0:1784948785.780818:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948785.780826:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948785.780833:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003ed79484. 00000800:00000200:1.0:1784948785.780841:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948785.780849:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948785.780854:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003ed79484 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784948785.780967:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948785.781016:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948785.781021:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003ed79484. 00000400:00000200:1.0:1784948785.781026:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948785.781033:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784948785.781048:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948785.781055:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948785.781058:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948785.781060:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948785.781063:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948785.781075:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784948785.781088:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x82e9 00000800:00000001:1.0:1784948785.781093:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948785.781101:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948785.781103:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948785.781106:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784948785.781110:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:1.0:1784948785.781112:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:1.0:1784948785.781115:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784948785.781118:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000db05a3db. 00000400:00000010:1.0:1784948785.781121:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:1.0:1784948785.781126:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948785.781128:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948786.419970:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948786.419977:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948786.419983:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948786.419984:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948786.420065:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948786.420068:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948786.420071:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948786.611970:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948786.611981:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948786.611983:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948786.611987:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948786.611989:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948786.611992:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948786.611994:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784948786.803969:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:2.0:1784948786.803973:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a12a46b5. 00000400:00000200:2.0:1784948786.803982:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948786.803987:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948786.803994:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11059 health :900 00000400:00000200:2.0:1784948786.803999:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948786.804003:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:2.0:1784948786.804007:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:3.0:1784948786.804056:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948786.804064:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:3.0:1784948786.804071:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948786.804073:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948786.804077:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000400:00000010:3.0:1784948786.804080:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000400:00000200:3.0:1784948786.804084:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948786.804088:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:3.0:1784948786.804091:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000307ddfde. 00000400:00000200:3.0:1784948786.804097:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000400:00000200:3.0:1784948786.804100:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000400:00000200:3.0:1784948786.804104:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00010000:00000001:2.0:1784948787.444032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948787.444042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948787.444051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948787.444053:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948787.444170:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948787.444177:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948787.444182:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948787.636043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948787.636058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948787.636061:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948787.636067:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948787.636071:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948787.636076:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948787.636079:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948787.828048:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948787.828061:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000db05a3db. 00000400:00000010:1.0:1784948787.828077:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:1.0:1784948787.828086:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000010:1.0:1784948787.828099:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e5d6f486. 00000400:00000200:1.0:1784948787.828108:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948787.828149:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948787.828157:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948787.828162:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3221:3221:244:4294967295] 192.168.202.137@tcp LPNI seq info [2095:2095:8:4294967295] 00000400:00000200:1.0:1784948787.828172:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948787.828181:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948787.828189:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948787.828198:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003ed79484. 00000800:00000200:1.0:1784948787.828206:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948787.828216:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948787.828222:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003ed79484 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948787.828245:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11061 health :900 00000800:00000200:0.0:1784948787.828412:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948787.828421:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003ed79484. 00000400:00000200:0.0:1784948787.828431:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948787.828442:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784948787.828446:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948787.828451:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948787.828458:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784948787.828465:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948787.828471:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948787.828478:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948787.828491:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948787.828509:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948787.828520:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948787.828533:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948787.828539:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3222:3222:244:4294967295] 192.168.202.137@tcp LPNI seq info [2096:2096:8:4294967295] 00000400:00000200:2.0:1784948787.828550:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948787.828559:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948787.828567:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000069a945c8. 00000800:00000200:2.0:1784948787.828576:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948787.828585:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948787.828590:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000069a945c8 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784948787.828745:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948787.828797:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948787.828803:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000069a945c8. 00000800:00000001:0.0:1784948787.828806:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948787.828810:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948787.828820:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948787.828821:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948787.828828:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948787.828834:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948787.828847:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000001:1.0:1784948787.828847:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948787.828871:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x82ed 00000800:00000001:0.0:1784948787.828880:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948787.828891:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948787.828894:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948787.828898:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948787.828938:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:0.0:1784948787.828942:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:0.0:1784948787.828946:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948787.828950:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000db05a3db. 00000400:00000010:0.0:1784948787.828954:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:0.0:1784948787.828962:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948787.828966:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948788.468042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948788.468051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948788.468060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948788.468062:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948788.468202:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948788.468209:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948788.468213:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948788.660013:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1784948788.660023:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948788.660029:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948788.660035:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948788.660036:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948788.660044:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948788.660047:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1784948788.660049:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005e9e18c4. 00000100:00000001:2.0:1784948788.660052:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.660058:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948788.660060:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948788.660067:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948788.660071:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948788.660079:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948788.660088:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003ed79484. 00000100:00000001:1.0:1784948788.660096:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948788.660107:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948788.660120:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948788.660123:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948788.660132:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e9e18c4] to pc [ptlrpcd_00_01+1] req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948788.660174:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948788.660177:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948788.660182:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948788.660186:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948788.660190:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948788.660195:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b47dc450. 00000020:00000040:1.0:1784948788.660200:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948788.660202:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948788.660203:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784948788.660208:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784948788.660209:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948788.660215:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784948788.660216:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948788.660216:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948788.660226:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948788.660226:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948788.660227:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000662152ce. 00000100:00000001:1.0:1784948788.660237:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.660249:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784948788.660250:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948788.660253:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948788.660262:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948788.660267:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948788.660270:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784948788.660273:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.660276:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948788.660281:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005e9e18c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289652608:0@lo:400: 00000100:00000001:2.0:1784948788.660288:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784948788.660290:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b47dc450] to pc [ptlrpcd_00_02+2] req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948788.660295:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948788.660301:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948788.660308:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784948788.660318:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 02000000:00000001:2.0:1784948788.660324:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948788.660335:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a16404b8. 00000100:00000001:1.0:1784948788.660337:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948788.660346:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000100:00000001:3.0:1784948788.660358:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784948788.660359:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289652608, portal 25 00000100:00000001:3.0:1784948788.660365:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948788.660367:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948788.660370:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948788.660373:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451652352 : -115544257899264 : ffff96e9c18d5500) 00000100:00000040:3.0:1784948788.660381:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948788.660390:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948788.660399:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948788.660403:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948788.660409:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948788.660414:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11065) 02000000:00000001:3.0:1784948788.660415:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948788.660417:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784948788.660425:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948788.660426:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948788.660435:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00100000:3.0:1784948788.660435:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b47dc450 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289652736:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948788.660439:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784948788.660445:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289652608, offset 0 00000100:00000001:0.0:1784948788.660446:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948788.660457:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948788.660462:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1784948788.660464:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784948788.660466:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948788.660470:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948788.660476:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a7af9ab9. 02000000:00000001:3.0:1784948788.660479:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948788.660484:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000400:00000200:2.0:1784948788.660489:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:3.0:1784948788.660492:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000100:00000200:3.0:1784948788.660500:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289652736, portal 10 00000100:00000001:3.0:1784948788.660502:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948788.660505:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451656832 : -115544257894784 : ffff96e9c18d6680) 00000400:00000200:2.0:1784948788.660512:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435d80 00000100:00000040:3.0:1784948788.660518:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948788.660532:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7951 [2] + 0 00000100:00000001:3.0:1784948788.660533:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948788.660542:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948788.660543:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007f1947c0. 00000400:00000010:3.0:1784948788.660549:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000400:00000200:2.0:1784948788.660551:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000003e35a545 00000100:00000200:3.0:1784948788.660562:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289652736, offset 0 00000400:00000200:2.0:1784948788.660563:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948788.660574:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948788.660580:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948788.660585:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003e35a545 00000400:00000010:2.0:1784948788.660590:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000003e35a545. 00000400:00000200:3.0:1784948788.660596:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948788.660601:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:3.0:1784948788.660603:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:2.0:1784948788.660606:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784948788.660612:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000222ac126 x1871651289652608 msgsize 224 00000400:00000200:3.0:1784948788.660613:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00100000:2.0:1784948788.660622:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784948788.660626:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1784948788.660628:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948788.660638:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948788.660646:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3223:3223:244:4294967295] 192.168.202.137@tcp LPNI seq info [2097:2097:8:4294967295] 00000100:00000001:2.0:1784948788.660664:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948788.660668:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948788.660682:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948788.660684:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:2.0:1784948788.660688:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000800:00000010:3.0:1784948788.660690:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000100:00000001:2.0:1784948788.660693:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948788.660701:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948788.660725:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784948788.660744:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784948788.660747:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:2.0:1784948788.660751:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948788.660757:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948788.660771:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.660774:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784948788.660782:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948788.660786:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.660787:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305728042 00000100:00000001:1.0:1784948788.660794:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784948788.660795:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2046 usecs (3 jiffies) 00000100:00000010:2.0:1784948788.660798:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000100:00100000:1.0:1784948788.660801:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289652608 00000100:00000001:3.0:1784948788.660803:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305728003 02000000:00000001:1.0:1784948788.660808:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948788.660811:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784948788.660813:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948788.660818:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948788.660824:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.660827:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305728042 00000100:00000001:3.0:1784948788.660828:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305728042 00000100:00100000:1.0:1784948788.660829:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289652608 00000100:00000010:2.0:1784948788.660837:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000010de4e21. 00000020:00000001:1.0:1784948788.660838:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948788.660839:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305728003 00000020:00000040:1.0:1784948788.660846:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784948788.660848:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1784948788.660852:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948788.660857:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305728042 00000100:00000001:3.0:1784948788.660857:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305728042 00000020:00000040:1.0:1784948788.660858:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784948788.660869:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:2.0:1784948788.660870:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001cae17e9. 00000100:00000001:3.0:1784948788.660870:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305728003 00000100:00000001:2.0:1784948788.660882:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948788.660890:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305728042 00000100:00000001:3.0:1784948788.660892:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305728042 00000100:00000010:2.0:1784948788.660895:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008a834f68. 00000020:00000001:1.0:1784948788.660898:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948788.660899:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305728003 00000100:00000001:2.0:1784948788.660902:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948788.660952:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948788.660955:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948788.660956:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305728042 00000020:00000010:1.0:1784948788.660960:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001690d9bc. 00000100:00000001:3.0:1784948788.660965:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948788.660967:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.660968:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948788.660970:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.660976:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948788.660980:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001690d9bc. 00000100:00000001:3.0:1784948788.660981:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.660982:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948788.660992:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948788.660996:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1784948788.660997:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a5bcc72e. 00000100:00000001:3.0:1784948788.661000:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948788.661001:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:1.0:1784948788.661005:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009adcfc93. 00000100:00000001:3.0:1784948788.661007:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948788.661009:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948788.661015:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1784948788.661017:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:3.0:1784948788.661017:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948788.661020:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948788.661022:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948788.661029:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948788.661030:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948788.661032:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948788.661039:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.661041:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948788.661046:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948788.661049:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.661059:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.661073:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948788.661091:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948788.661094:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948788.661101:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:1.0:1784948788.661101:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 704 00000100:00000001:0.0:1784948788.661108:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:1.0:1784948788.661110:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948788.661112:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.661116:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165125200 : -115539544426416 : ffff96eada7f3850) 00000100:00000040:1.0:1784948788.661123:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000222ac126 x1871651289652608/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:449/0 lens 224/0 e 0 to 0 dl 1784948799 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948788.661133:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948788.661137:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001690d9bc. 00000100:00000001:1.0:1784948788.661140:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948788.661142:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001690d9bc. 00000100:00100000:1.0:1784948788.661148:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000222ac126 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289652608:12345-0@lo:400: 00000100:00000200:1.0:1784948788.661152:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289652608 00000020:00000001:1.0:1784948788.661156:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948788.661162:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948788.661163:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948788.661169:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948788.661173:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948788.661176:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948788.661180:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948788.661182:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948788.661185:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948788.661188:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948788.661191:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948788.661195:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000200:3.0:1784948788.661200:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948788.661201:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948788.661207:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000010:3.0:1784948788.661208:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 02000000:00000010:1.0:1784948788.661213:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fed721f2. 00000400:00000200:3.0:1784948788.661216:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784948788.661218:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.661223:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948788.661226:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784948788.661229:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948788.661235:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289652608 00000400:00000200:3.0:1784948788.661235:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000400:00000010:3.0:1784948788.661241:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00010000:00000001:1.0:1784948788.661243:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948788.661248:0:337114:0:(events.c:34:request_out_callback()) Process entered 00010000:00000200:1.0:1784948788.661252:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000222ac126 x1871651289652608/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:449/0 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948788.661259:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784948788.661260:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948788.661278:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948788.661287:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:1.0:1784948788.661290:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948788.661295:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1784948788.661297:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948788.661297:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948788.661305:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:1.0:1784948788.661311:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=110 v=5 (1 1 1 1) 00000400:00000200:2.0:1784948788.661324:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948788.661328:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.661329:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784948788.661335:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784948788.661339:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:2.0:1784948788.661348:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435e00 00000100:00000001:1.0:1784948788.661350:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948788.661361:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948788.661365:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948788.661371:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 153360 00000100:00000001:1.0:1784948788.661373:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948788.661381:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948788.661387:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000800:00000001:2.0:1784948788.661389:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:1.0:1784948788.661393:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289652608, offset 224 00000400:00000200:1.0:1784948788.661400:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:2.0:1784948788.661403:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948788.661410:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948788.661411:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784948788.661420:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948788.661427:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435d80 00000100:00000001:2.0:1784948788.661428:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948788.661434:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:1.0:1784948788.661439:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x82f1 [1] + 224 00000100:00000010:2.0:1784948788.661442:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005db4bd4e. 00000100:00000040:2.0:1784948788.661447:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000005db4bd4e x1871651289652736 msgsize 224 00000400:00000200:1.0:1784948788.661449:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:2.0:1784948788.661458:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:1.0:1784948788.661463:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948788.661470:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948788.661474:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948788.661481:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784948788.661486:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948788.661496:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948788.661505:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948788.661513:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948788.661526:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000100:00000001:0.0:1784948788.661528:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:1.0:1784948788.661529:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000100:00100000:0.0:1784948788.661535:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289652736 00000100:00000001:1.0:1784948788.661537:0:342756:0:(events.c:386:reply_out_callback()) Process entered 02000000:00000001:0.0:1784948788.661539:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948788.661541:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948788.661544:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948788.661548:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948788.661549:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948788.661552:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948788.661558:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948788.661559:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289652736 00000100:00000040:1.0:1784948788.661559:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784948788.661566:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784948788.661567:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00010000:00000001:1.0:1784948788.661568:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000040:0.0:1784948788.661573:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948788.661574:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784948788.661574:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1784948788.661580:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000001:3.0:1784948788.661581:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784948788.661582:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000040:0.0:1784948788.661585:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000010:3.0:1784948788.661588:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a16404b8. 00000100:00000040:1.0:1784948788.661593:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000222ac126 x1871651289652608/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:449/0 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948788.661594:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784948788.661597:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000020:00000001:0.0:1784948788.661602:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000010:3.0:1784948788.661604:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000100:00000001:0.0:1784948788.661613:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948788.661614:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784948788.661620:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000222ac126 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289652608:12345-0@lo:400: Request processed in 484us (1004us total) trans 0 rc 0/0 00000020:00000010:0.0:1784948788.661622:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ca00af7. 00000100:00000200:3.0:1784948788.661625:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948788.661631:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948788.661641:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ca00af7. 00000100:00100000:1.0:1784948788.661649:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 704 00000020:00000010:0.0:1784948788.661652:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000100:00000040:1.0:1784948788.661655:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:3.0:1784948788.661656:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948788.661661:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000030e14bca. 00000100:00000001:1.0:1784948788.661663:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948788.661668:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948788.661671:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000020:00000010:1.0:1784948788.661677:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009adcfc93. 00000100:00000040:0.0:1784948788.661683:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948788.661689:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000010:1.0:1784948788.661690:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000100:00000001:3.0:1784948788.661690:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948788.661692:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948788.661696:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948788.661699:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948788.661700:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 00000100:00000001:3.0:1784948788.661701:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948788.661726:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948788.661729:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948788.661731:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948788.661753:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948788.661754:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948788.661756:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948788.661757:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948788.661763:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948788.661767:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784948788.661770:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000fed721f2. 00000100:00001000:3.0:1784948788.661772:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=110 v=5 (5 5 5 5) 02000000:00000001:1.0:1784948788.661774:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948788.661775:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948788.661777:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784948788.661788:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=110 v=5 (1 1 1 1) 00000100:00000200:1.0:1784948788.661789:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948788.661793:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948788.661798:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:1.0:1784948788.661800:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d40cdb47. 00000100:00000001:3.0:1784948788.661802:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00100000:0.0:1784948788.661809:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 680 00000100:00000001:3.0:1784948788.661810:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948788.661811:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000341aff9b. 00000100:00000040:0.0:1784948788.661815:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00010000:00000001:3.0:1784948788.661819:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784948788.661820:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470023296 : -115544239528320 : ffff96e9c2a5a680) 00010000:00000001:3.0:1784948788.661825:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948788.661831:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005db4bd4e x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:449/0 lens 224/0 e 0 to 0 dl 1784948799 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948788.661832:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948788.661847:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948788.661850:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948788.661856:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ca00af7. 00000100:00000001:3.0:1784948788.661859:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948788.661863:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948788.661865:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948788.661870:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948788.661871:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ca00af7. 00000100:00000040:3.0:1784948788.661881:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948788.661886:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005db4bd4e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289652736:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948788.661901:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289652736 00000100:00100000:3.0:1784948788.661908:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005e9e18c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289652608:0@lo:400: 00000100:00000001:3.0:1784948788.661916:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948788.661918:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784948788.661935:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948788.661939:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948788.661940:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948788.661943:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948788.661945:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948788.661948:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948788.661952:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948788.661958:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:3.0:1784948788.661961:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005e9e18c4 x1871651289652608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948788.661962:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948788.661967:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948788.661974:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948788.661980:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948788.661981:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948788.661985:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:3.0:1784948788.661987:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948788.661991:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009e327039. 02000000:00000010:3.0:1784948788.661991:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:0.0:1784948788.661999:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948788.662002:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784948788.662004:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948788.662006:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:0.0:1784948788.662008:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948788.662011:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00010000:00000040:0.0:1784948788.662014:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289652736 00000020:00000001:3.0:1784948788.662016:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000001:0.0:1784948788.662019:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000010:3.0:1784948788.662021:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003ed79484. 00010000:00000200:0.0:1784948788.662026:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005db4bd4e x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:449/0 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948788.662027:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948788.662031:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948788.662035:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005e9e18c4. 00000100:00000001:3.0:1784948788.662043:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00010000:00000001:0.0:1784948788.662045:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948788.662046:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948788.662051:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948788.662052:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784948788.662060:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=105 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948788.662063:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948788.662067:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784948788.662067:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948788.662073:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948788.662074:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948788.662078:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948788.662080:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948788.662084:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948788.662085:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948788.662088:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948788.662091:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948788.662094:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000100:00000200:0.0:1784948788.662099:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289652736, offset 224 00000400:00000200:0.0:1784948788.662104:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948788.662111:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948788.662116:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3224:3224:244:4294967295] 192.168.202.137@tcp LPNI seq info [2098:2098:8:4294967295] 00000400:00000200:0.0:1784948788.662125:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948788.662130:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948788.662133:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f64a2f6c. 00000800:00000200:0.0:1784948788.662138:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948788.662143:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948788.662146:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f64a2f6c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948788.662162:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948788.662165:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948788.662167:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948788.662169:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948788.662171:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948788.662174:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005db4bd4e x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:449/0 lens 224/224 e 0 to 0 dl 1784948799 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948788.662183:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005db4bd4e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289652736:12345-192.168.202.137@tcp:400: Request processed in 334us (734us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948788.662190:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 680 00000100:00000040:0.0:1784948788.662192:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948788.662195:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948788.662196:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948788.662199:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000030e14bca. 00000020:00000010:0.0:1784948788.662203:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948788.662207:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 00000020:00000040:0.0:1784948788.662211:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948788.662214:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948788.662307:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948788.662313:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f64a2f6c. 00000400:00000200:2.0:1784948788.662320:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948788.662327:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948788.662329:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948788.662338:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00000800:00000001:3.0:1784948788.662338:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948788.662345:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000800:00000001:3.0:1784948788.662345:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948788.662350:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948788.662355:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:3.0:1784948788.662361:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948788.662362:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948788.662377:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435e00 00000400:00000200:3.0:1784948788.662384:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x82f9 [1] + 224 00000800:00000001:3.0:1784948788.662390:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948788.662398:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948788.662400:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948788.662404:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948788.662407:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948788.662410:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948788.662419:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948788.662437:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948788.662442:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948788.662444:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948788.662474:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948788.662478:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948788.662482:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948788.662484:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948788.662486:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000400:00000200:0.0:1784948788.662493:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000400:00000010:0.0:1784948788.662494:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000100:00000001:0.0:1784948788.662497:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948788.662502:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948788.662511:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948788.662518:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948788.662519:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948788.662521:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948788.662523:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948788.662525:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948788.662526:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948788.662528:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948788.662532:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=105 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948788.662536:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=105 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948788.662538:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948788.662539:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948788.662541:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948788.662544:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948788.662550:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948788.662553:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948788.662554:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948788.662556:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948788.662557:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948788.662560:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948788.662566:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948788.662568:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948788.662569:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948788.662572:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948788.662578:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b47dc450 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289652736:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948788.662582:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948788.662583:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948788.662585:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b47dc450 x1871651289652736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948788.662597:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948788.662599:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948788.662600:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a7af9ab9. 02000000:00000001:0.0:1784948788.662604:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948788.662606:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948788.662607:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948788.662609:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948788.662610:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000662152ce. 02000000:00000001:0.0:1784948788.662613:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948788.662614:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948788.662615:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b47dc450. 00000100:00000001:0.0:1784948788.662618:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948788.662619:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948788.662621:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948788.662624:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948788.662625:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948788.662627:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948788.662629:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948788.703002:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305728042, at 4305728043 00000100:00000040:0.0:1784948788.703013:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305728042, at 4305728043 00000100:00000010:2.0:1784948788.703016:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00000100:00000040:1.0:1784948788.703023:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305728042, at 4305728043 00000100:00000010:0.0:1784948788.703031:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00000100:00000010:1.0:1784948788.703037:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000010de4e21. 00000100:00000040:3.0:1784948788.703995:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305728042, at 4305728044 00000100:00000010:3.0:1784948788.704006:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008a834f68. 00000800:00000100:2.0:1784948788.852043:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000800:00000010:2.0:1784948788.852051:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f538dcf6. 00000400:00000200:2.0:1784948788.852061:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948788.852069:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948788.852079:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11061 health :900 00000400:00000200:2.0:1784948788.852087:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948788.852092:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:2.0:1784948788.852097:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:3.0:1784948788.852171:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948788.852183:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:3.0:1784948788.852191:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948788.852194:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948788.852198:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:3.0:1784948788.852201:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000400:00000200:3.0:1784948788.852207:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948788.852212:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:3.0:1784948788.852217:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000cf77ef26. 00000400:00000200:3.0:1784948788.852224:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:3.0:1784948788.852227:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:3.0:1784948788.852232:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00010000:00000001:2.0:1784948789.491975:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948789.491981:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948789.491986:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948789.491989:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948789.491991:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948789.491993:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948789.491995:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948789.491997:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948789.491999:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948789.492000:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948789.492078:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948789.492082:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948789.492085:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948789.683983:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948789.683997:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948789.683999:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948789.684005:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948789.684008:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948789.684011:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948789.684014:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948789.876003:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:2.0:1784948789.876005:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.2@tcp 00000800:00000010:2.0:1784948789.876010:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f579e76d. 00000400:00000010:1.0:1784948789.876012:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a007df80. 00000400:00000200:2.0:1784948789.876017:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:1.0:1784948789.876020:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000200:2.0:1784948789.876025:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784948789.876027:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000400:00000200:2.0:1784948789.876032:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11062 health :800 00000400:00000010:1.0:1784948789.876037:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000acbf6818. 00000400:00000200:2.0:1784948789.876038:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948789.876046:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:1.0:1784948789.876048:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948789.876052:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:1.0:1784948789.876092:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948789.876102:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948789.876108:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3225:3225:245:4294967295] 192.168.202.137@tcp LPNI seq info [2099:2099:8:4294967295] 00000400:00000200:1.0:1784948789.876117:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948789.876126:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948789.876132:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948789.876138:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000036e798e7. 00000400:00000200:3.0:1784948789.876143:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000100:2.0:1784948789.876152:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000800:00000200:1.0:1784948789.876153:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948789.876159:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000800:00000010:2.0:1784948789.876162:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000071cce8cf. 00000800:00000200:1.0:1784948789.876172:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:2.0:1784948789.876172:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000001:3.0:1784948789.876174:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948789.876182:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1784948789.876184:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000036e798e7 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784948789.876188:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948789.876189:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000400:00000010:3.0:1784948789.876197:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000400:00000200:3.0:1784948789.876205:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948789.876206:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948789.876213:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:2.0:1784948789.876215:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000010:3.0:1784948789.876224:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eb3e5fa2. 00000400:00000200:3.0:1784948789.876233:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-110 00000400:00000200:3.0:1784948789.876239:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -110 00000400:00000200:2.0:1784948789.876243:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:3.0:1784948789.876255:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:3.0:1784948789.876271:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000001:3.0:1784948789.876276:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948789.876279:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948789.876282:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000400:00000010:3.0:1784948789.876284:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000400:00000200:3.0:1784948789.876289:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948789.876294:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:3.0:1784948789.876299:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007a6587e5. 00000400:00000200:3.0:1784948789.876308:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000200:3.0:1784948789.876313:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:3.0:1784948789.876317:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000800:00000200:0.0:1784948789.876454:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948789.876459:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000036e798e7. 00000400:00000200:0.0:1784948789.876465:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948789.876466:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948789.876473:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948789.876478:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:2.0:1784948789.876478:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948789.876483:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948789.876484:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784948789.876487:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948789.876494:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948789.876505:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948789.876512:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948789.876519:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948789.876522:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3226:3226:247:4294967295] 192.168.202.137@tcp LPNI seq info [2100:2100:8:4294967295] 00000400:00000200:2.0:1784948789.876528:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948789.876531:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948789.876534:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000069a945c8. 00000800:00000200:2.0:1784948789.876538:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948789.876541:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948789.876543:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000069a945c8 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784948789.876573:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948789.876613:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948789.876616:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000069a945c8. 00000400:00000200:2.0:1784948789.876619:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948789.876622:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948789.876630:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948789.876634:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948789.876636:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948789.876639:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948789.876639:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948789.876647:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948789.876653:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x830d 00000800:00000001:2.0:1784948789.876657:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948789.876661:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948789.876662:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948789.876664:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948789.876666:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000400:00000010:2.0:1784948789.876668:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000400:00000200:2.0:1784948789.876669:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948789.876671:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a007df80. 00000400:00000010:2.0:1784948789.876674:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000800:00000001:2.0:1784948789.876678:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948789.876680:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948790.516018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948790.516026:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948790.516032:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948790.516033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948790.516138:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948790.516143:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948790.516147:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948790.708023:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948790.708038:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948790.708040:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948790.708046:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948790.708049:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948790.708054:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948790.708057:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784948790.900030:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.6@tcp 00000800:00000010:2.0:1784948790.900038:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000efb7ac25. 00000400:00000200:2.0:1784948790.900050:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948790.900058:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948790.900069:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11063 health :900 00000400:00000200:2.0:1784948790.900077:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948790.900082:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:2.0:1784948790.900087:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000800:00000100:2.0:1784948790.900118:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000800:00000010:2.0:1784948790.900122:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b4c31151. 00000400:00000200:2.0:1784948790.900129:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948790.900133:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948790.900137:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948790.900142:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:2.0:1784948790.900146:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:3.0:1784948790.900169:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948790.900184:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 00000400:00000001:3.0:1784948790.900193:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948790.900196:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948790.900200:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000400:00000010:3.0:1784948790.900203:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 00000400:00000200:3.0:1784948790.900209:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948790.900214:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:3.0:1784948790.900219:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f0e4e38. 00000400:00000200:3.0:1784948790.900227:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-110 00000400:00000200:3.0:1784948790.900230:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -110 00000400:00000200:3.0:1784948790.900235:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:3.0:1784948790.900239:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:3.0:1784948790.900242:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948790.900244:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948790.900245:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784948790.900247:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000400:00000200:3.0:1784948790.900250:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948790.900252:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:3.0:1784948790.900255:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000eae26e4. 00000400:00000200:3.0:1784948790.900259:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:3.0:1784948790.900262:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:3.0:1784948790.900265:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00010000:00000001:2.0:1784948791.539985:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948791.539992:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948791.539999:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948791.540000:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948791.540088:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948791.540096:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948791.540101:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948791.731970:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948791.731984:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948791.731986:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948791.731992:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948791.731995:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948791.731999:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948791.732001:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948791.923970:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948791.923978:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000090a03b90. 00000800:00000100:2.0:1784948791.923982:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.10@tcp 00000400:00000010:1.0:1784948791.923986:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000800:00000010:2.0:1784948791.923987:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003264a2f8. 00000400:00000010:1.0:1784948791.923993:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000200:2.0:1784948791.923994:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948791.924001:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784948791.924002:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000451e3fe4. 00000400:00000200:1.0:1784948791.924009:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948791.924009:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11064 health :700 00000400:00000200:2.0:1784948791.924016:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948791.924019:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:1.0:1784948791.924025:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948791.924030:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948791.924033:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3227:3227:249:4294967295] 192.168.202.137@tcp LPNI seq info [2101:2101:8:4294967295] 00000400:00000200:1.0:1784948791.924038:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948791.924043:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948791.924049:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948791.924050:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000800:00000010:1.0:1784948791.924054:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000034aacccc. 00000800:00000200:1.0:1784948791.924061:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948791.924069:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948791.924073:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784948791.924088:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948791.924096:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:3.0:1784948791.924100:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948791.924102:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948791.924104:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948791.924106:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000400:00000200:3.0:1784948791.924109:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948791.924112:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:3.0:1784948791.924115:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001879354a. 00000400:00000200:3.0:1784948791.924118:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-110 00000400:00000200:3.0:1784948791.924120:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -110 00000400:00000200:3.0:1784948791.924122:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000800:00000200:0.0:1784948791.924231:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948791.924236:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000034aacccc. 00000800:00000001:3.0:1784948791.924240:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948791.924241:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948791.924247:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948791.924250:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948791.924253:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948791.924253:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948791.924258:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948791.924263:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948791.924268:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948791.924278:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784948791.924285:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784948791.924292:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948791.924295:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3228:3228:250:4294967295] 192.168.202.137@tcp LPNI seq info [2102:2102:8:4294967295] 00000400:00000200:3.0:1784948791.924301:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784948791.924304:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948791.924307:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784948791.924311:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948791.924314:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948791.924316:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784948791.924351:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784948791.924389:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948791.924392:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:3.0:1784948791.924396:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948791.924397:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948791.924401:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948791.924406:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948791.924409:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948791.924411:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948791.924414:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948791.924421:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948791.924430:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8311 00000800:00000001:0.0:1784948791.924434:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948791.924444:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948791.924446:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948791.924448:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948791.924451:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:0.0:1784948791.924452:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000400:00000200:0.0:1784948791.924454:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948791.924456:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000090a03b90. 00000400:00000010:0.0:1784948791.924458:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:0.0:1784948791.924462:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948791.924463:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948792.564029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948792.564038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948792.564045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948792.564047:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948792.564177:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948792.564183:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948792.564188:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948792.756113:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948792.756129:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948792.756133:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948792.756141:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948792.756145:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948792.756149:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948792.756152:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948793.588025:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948793.588035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948793.588043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948793.588046:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948793.588128:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948793.588135:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948793.588140:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948793.779995:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948793.780003:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784948793.780016:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948768/real 1784948768] req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948793 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948793.780037:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.780040:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948793.780043:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000010aeeaab. 00000400:00000200:1.0:1784948793.780051:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000100:00000040:3.0:1784948793.780054:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1784948793.780055:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000100:00000001:1.0:1784948793.780062:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784948793.780063:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000200:1.0:1784948793.780070:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948793 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948793.780075:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fa1e143a. 00000020:00000040:3.0:1784948793.780085:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000200:1.0:1784948793.780086:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948793 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948793.780094:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784948793.780098:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1784948793.780099:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948793.780103:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948793.780107:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.780108:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948793.780111:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948793.780117:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948793 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784948793.780119:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:1.0:1784948793.780138:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.780143:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.780147:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948793.780151:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948793.780155:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948793.780157:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948793.780162:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948793.780167:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.780177:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1784948793.780181:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948793 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948793.780183:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948793.780196:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fa1e143a] to pc [ptlrpcd_00_03+3] req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948793.780197:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784948793.780207:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948793.780214:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948793.780217:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948793.780220:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784948793.780223:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784948793.780227:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784948793.780246:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:1.0:1784948793.780250:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1784948793.780251:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948793.780256:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000040:1.0:1784948793.780257:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948793 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1784948793.780268:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948793.780271:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784948793.780274:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784948793.780278:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784948793.780285:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11040 00000100:00000001:2.0:1784948793.780288:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948793.780294:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289651584:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784948793.780298:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784948793.780302:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784948793.780305:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1784948793.780311:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11039 00000100:00000040:2.0:1784948793.780319:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948793.780329:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000400:00000200:3.0:1784948793.780340:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000040:3.0:1784948793.780348:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948793.780349:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948793.780352:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948793.780355:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948793.780359:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948793.780364:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:2.0:1784948793.780366:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1784948793.780367:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948793.780372:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:2.0:1784948793.780374:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948793.780375:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948793.780392:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:2.0:1784948793.780394:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948793.780396:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:3.0:1784948793.780403:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00100000:2.0:1784948793.780407:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fa1e143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289652864:0@lo:400: 00000100:00000040:3.0:1784948793.780409:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948793.780416:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948793.780422:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1784948793.780422:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948793.780428:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:3.0:1784948793.780431:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:2.0:1784948793.780434:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1784948793.780435:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000001:2.0:1784948793.780441:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948793.780442:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:3.0:1784948793.780451:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948793.780452:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1784948793.780454:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1784948793.780456:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784948793.780463:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000f1904ca. 00000100:00000001:1.0:1784948793.780464:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784948793.780465:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948793.780473:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948793.780473:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948793.780478:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008d623923 x1871651289651584/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948793 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948793.780481:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1784948793.780486:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948793.780488:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009129b91b. 00000100:00000010:3.0:1784948793.780495:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b13d185. 00000020:00000040:3.0:1784948793.780502:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=4 obd=lustre-OST0000-osc-MDT0000 00000400:00000010:2.0:1784948793.780503:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 02000000:00000001:3.0:1784948793.780509:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784948793.780512:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.780515:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1784948793.780517:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289652864, portal 25 02000000:00000001:3.0:1784948793.780519:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948793.780520:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948793.780523:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948793.780526:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1784948793.780529:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1784948793.780532:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482520704 : -115544227030912 : ffff96e9c3645880) 02000000:00000010:3.0:1784948793.780533:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000900bf136. 02000000:00000010:1.0:1784948793.780535:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006107fe19. 00000100:00000001:3.0:1784948793.780572:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948793.780574:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:2.0:1784948793.780578:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948793.780581:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948793.780586:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00100000:3.0:1784948793.780588:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948793.780594:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948793.780595:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:0.0:1784948793.780604:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948793.780605:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000009b7d615c. 02000000:00000001:1.0:1784948793.780617:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948793.780621:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948793.780626:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008d623923. 00000100:00000001:2.0:1784948793.780629:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948793.780631:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948793.780634:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948793.780635:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.780640:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948793.780644:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784948793.780645:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948793.780647:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b13d185] to pc [ptlrpcd_rcv+-1] req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948793.780654:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000001:1.0:1784948793.780655:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948793.780659:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.780712:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948793.780713:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948793.780721:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289652864, offset 0 00000400:00000200:2.0:1784948793.780735:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948793.780738:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948793.780744:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948793.780750:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:2.0:1784948793.780758:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000010:3.0:1784948793.780758:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000092889a66. 00000020:00000040:3.0:1784948793.780766:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:1.0:1784948793.780768:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:3.0:1784948793.780776:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1784948793.780779:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000400:00000200:2.0:1784948793.780784:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6435e80 02000000:00000001:3.0:1784948793.780784:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948793.780790:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948793.780791:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 02000000:00000010:3.0:1784948793.780798:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000400:00000010:1.0:1784948793.780799:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000400:00000200:2.0:1784948793.780800:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7971 [2] + 0 00000100:00000001:3.0:1784948793.780808:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948793.780811:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f5a5b57b. 00000100:00000040:3.0:1784948793.780818:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784948793.780821:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bc3ccb92 00000100:00000001:3.0:1784948793.780832:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1784948793.780835:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009234a5fd. 00000100:00000001:3.0:1784948793.780835:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948793.780838:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784948793.780842:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092889a66] to pc [ptlrpcd_00_00+0] req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948793.780849:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000400:00000200:2.0:1784948793.780854:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948793.780867:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000400:00000200:1.0:1784948793.780879:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.780885:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.780890:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948793.780890:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948793.780898:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000040:3.0:1784948793.780899:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11070) 00000400:00000200:1.0:1784948793.780940:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3229:3229:250:4294967295] 1.2.3.4@tcp LPNI seq info [97:97:8:4294967295] 00000400:00000200:1.0:1784948793.780960:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000040:3.0:1784948793.780970:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11070) 00000400:00000200:1.0:1784948793.780974:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.780982:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:3.0:1784948793.780986:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784948793.780988:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc3ccb92 00000800:00000010:1.0:1784948793.780989:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002a753d9d. 00000100:00000001:3.0:1784948793.780992:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1784948793.780998:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bc3ccb92. 00000100:00000001:3.0:1784948793.780998:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000800:00000200:1.0:1784948793.781006:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:3.0:1784948793.781007:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948793.781011:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:1.0:1784948793.781014:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 10 secs later) 00000100:00000200:2.0:1784948793.781018:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784948793.781030:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000bc66e638 x1871651289652864 msgsize 224 00000800:00000200:1.1:1784948793.781033:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948793.781043:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 10 secs later) 00000100:00000001:3.0:1784948793.781047:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948793.781052:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948793.781053:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000100:1.0:1784948793.781059:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 02000000:00000001:3.0:1784948793.781060:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1784948793.781062:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000010:1.0:1784948793.781063:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002a753d9d. 02000000:00000001:3.0:1784948793.781064:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948793.781072:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948793.781073:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1784948793.781079:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000092889a66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289653120:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948793.781085:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948793.781089:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000001:3.0:1784948793.781092:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948793.781096:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948793.781098:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948793.781100:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784948793.781102:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948793.781103:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 02000000:00000010:3.0:1784948793.781105:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a7af9ab9. 00000400:00000200:1.0:1784948793.781112:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000010:2.0:1784948793.781112:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 02000000:00000001:3.0:1784948793.781112:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948793.781124:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e8e1fecb. 00000100:00000001:2.0:1784948793.781126:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1784948793.781127:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:3.0:1784948793.781134:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000200:2.0:1784948793.781140:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948793.781142:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289653120, portal 10 00000400:00000200:1.0:1784948793.781144:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:3.0:1784948793.781151:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784948793.781153:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000001:3.0:1784948793.781156:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482529664 : -115544227021952 : ffff96e9c3647b80) 00000100:00000001:0.0:1784948793.781157:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948793.781166:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:1.0:1784948793.781173:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1784948793.781174:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948793.781178:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784948793.781181:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948793.781184:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:1.0:1784948793.781190:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000100:00000040:2.0:1784948793.781194:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948793.781196:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948793.781198:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:3.0:1784948793.781201:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784948793.781208:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000040:3.0:1784948793.781212:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948793.781217:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000400:00000010:3.0:1784948793.781220:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000200:3.0:1784948793.781229:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289653120, offset 0 00000400:00000200:1.0:1784948793.781232:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:2.0:1784948793.781239:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948793.781242:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000400:00000200:3.0:1784948793.781249:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948793.781250:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784948793.781256:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:2.0:1784948793.781269:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948793.781276:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948793.781280:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305733163 00000400:00000200:3.0:1784948793.781283:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948793.781291:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000010:2.0:1784948793.781294:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001cae17e9. 00000400:00000200:3.0:1784948793.781294:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948793.781301:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948793.781306:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3230:3230:250:4294967295] 192.168.202.137@tcp LPNI seq info [2103:2103:8:4294967295] 00000100:00000001:2.0:1784948793.781307:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784948793.781323:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305733163 00000400:00000200:3.0:1784948793.781325:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948793.781335:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:2.0:1784948793.781335:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000445487c0. 00000800:00000200:3.0:1784948793.781340:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948793.781346:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000800:00000010:3.0:1784948793.781347:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000100:00000040:0.0:1784948793.781348:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948793.781359:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305733163 00000800:00000200:3.0:1784948793.781365:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000010:2.0:1784948793.781373:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000800:00000200:3.0:1784948793.781381:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948793.781390:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948793.781391:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948793.781393:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00100000:1.0:1784948793.781398:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289652864 02000000:00000001:1.0:1784948793.781402:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948793.781405:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305733163 00000100:00000001:1.0:1784948793.781406:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948793.781410:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.781410:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948793.781420:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2047 usecs (3 jiffies) 02000000:00000001:1.0:1784948793.781421:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948793.781424:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948793.781432:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305733124 00000100:00000001:0.0:1784948793.781433:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948793.781434:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289652864 00000100:00000001:3.0:1784948793.781443:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305733163 00000020:00000001:1.0:1784948793.781444:0:342757:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1784948793.781445:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948793.781448:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305733124 02000000:00000001:0.0:1784948793.781462:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784948793.781463:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784948793.781463:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305733163 00000020:00000001:1.0:1784948793.781473:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:0.0:1784948793.781474:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.781474:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305733124 00000020:00000040:1.0:1784948793.781481:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948793.781482:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305733163 00000100:00100000:0.0:1784948793.781489:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b13d185 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289652992:1.2.3.4@tcp:8: 00000020:00000001:1.0:1784948793.781492:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948793.781493:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305733124 00000100:00000010:3.0:1784948793.781503:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fd993a25. 00000100:00000001:0.0:1784948793.781505:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1784948793.781505:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948793.781511:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948793.781513:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.781517:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948793.781519:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948793.781521:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.781525:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948793.781526:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:0.0:1784948793.781530:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.781533:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.781536:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948793.781539:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.781540:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948793.781542:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00000001:3.0:1784948793.781552:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948793.781554:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948793.781556:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:1.0:1784948793.781557:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000100:00000001:0.0:1784948793.781560:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.781583:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1784948793.781586:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009adcfc93. 00000100:00000001:3.0:1784948793.781587:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948793.781589:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948793.781596:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948793.781597:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1784948793.781598:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:0.0:1784948793.781603:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.781605:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948793.781612:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948793.781616:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305733124 00000100:00000040:1.0:1784948793.781617:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948793.781624:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948793.781630:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948793.781630:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948793.781634:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948793.781637:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.781637:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948793.781642:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948793.781651:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.781653:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948793.781655:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948793.781711:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948793.781715:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948793.781721:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948793.781727:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948793.781731:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948793.781732:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948793.781736:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948793.781738:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948793.781739:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948793.781744:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948793.781747:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 705 00000100:00000001:2.0:1784948793.781750:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948793.781753:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948793.781758:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.781759:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165112912 : -115539544438704 : ffff96eada7f0850) 00000100:00000001:0.0:1784948793.781767:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948793.781772:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bc66e638 x1871651289652864/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:454/0 lens 224/0 e 0 to 0 dl 1784948804 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948793.781782:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948793.781786:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948793.781790:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948793.781793:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00100000:1.0:1784948793.781797:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bc66e638 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289652864:12345-0@lo:400: 00000100:00000200:1.0:1784948793.781802:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289652864 00000020:00000001:1.0:1784948793.781805:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948793.781810:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948793.781813:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948793.781815:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:1.0:1784948793.781816:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000010:3.0:1784948793.781823:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000020:00000001:1.0:1784948793.781824:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948793.781832:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:3.0:1784948793.781834:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784948793.781835:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948793.781838:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948793.781842:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:3.0:1784948793.781846:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784948793.781848:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784948793.781854:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000020:00000001:1.0:1784948793.781855:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948793.781858:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948793.781861:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000020:00000001:1.0:1784948793.781862:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:2.0:1784948793.781867:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948793.781869:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948793.781870:0:337114:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1784948793.781875:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:2.0:1784948793.781877:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948793.781880:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784948793.781883:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fed721f2. 02000000:00000001:1.0:1784948793.781890:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.781897:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948793.781901:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948793.781942:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948793.781946:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948793.781951:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948793.781960:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289652864 00000100:00000001:3.0:1784948793.781963:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948793.781965:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948793.781967:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784948793.781974:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6435f80 00010000:00000200:1.0:1784948793.781975:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bc66e638 x1871651289652864/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:454/0 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948793.781991:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948793.781994:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948793.781997:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 153584 00000100:00001000:1.0:1784948793.782000:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=115 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948793.782013:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:2.0:1784948793.782016:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784948793.782017:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948793.782024:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948793.782027:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948793.782029:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948793.782032:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:2.0:1784948793.782033:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948793.782035:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:2.0:1784948793.782042:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1784948793.782043:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000100:00000200:1.0:1784948793.782051:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289652864, offset 224 00000400:00000200:2.0:1784948793.782058:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948793.782060:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948793.782073:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784948793.782074:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948793.782080:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:1.0:1784948793.782084:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6435e80 00000100:00000010:2.0:1784948793.782090:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000049907b7f. 00000400:00000200:1.0:1784948793.782091:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8315 [1] + 224 00000100:00000040:2.0:1784948793.782100:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000049907b7f x1871651289653120 msgsize 224 00000400:00000200:1.0:1784948793.782102:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948793.782113:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948793.782120:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784948793.782124:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:1.0:1784948793.782127:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948793.782140:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948793.782152:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948793.782157:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948793.782165:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948793.782168:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:1.0:1784948793.782172:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000800:00000001:2.0:1784948793.782176:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948793.782177:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948793.782181:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948793.782189:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948793.782194:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948793.782195:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:1.0:1784948793.782197:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00100000:0.0:1784948793.782201:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289653120 00000020:00000001:1.0:1784948793.782202:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948793.782207:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948793.782207:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948793.782208:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948793.782215:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948793.782215:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948793.782216:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bc66e638 x1871651289652864/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:454/0 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948793.782221:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784948793.782225:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1784948793.782231:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784948793.782233:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948793.782239:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009129b91b. 00000100:00100000:1.0:1784948793.782240:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bc66e638 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289652864:12345-0@lo:400: Request processed in 455us (1207us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948793.782242:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289653120 00000020:00000001:0.0:1784948793.782251:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1784948793.782252:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000020:00000040:0.0:1784948793.782257:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00100000:1.0:1784948793.782258:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 705 00000400:00000010:2.0:1784948793.782260:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000020:00000001:0.0:1784948793.782266:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1784948793.782267:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784948793.782269:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:0.0:1784948793.782272:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948793.782280:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:2.0:1784948793.782281:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948793.782289:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948793.782297:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948793.782302:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948793.782304:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948793.782311:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000020:00000010:1.0:1784948793.782312:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009adcfc93. 00000100:00000200:2.0:1784948793.782318:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948793.782320:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948793.782322:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:0.0:1784948793.782330:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000020:00000010:1.0:1784948793.782332:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000020:00000010:0.0:1784948793.782342:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000020:00000040:1.0:1784948793.782343:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784948793.782349:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784948793.782352:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007035a559. 02000000:00000001:1.0:1784948793.782353:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948793.782355:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784948793.782358:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000fed721f2. 00000020:00000010:0.0:1784948793.782360:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784948793.782363:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948793.782364:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784948793.782370:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948793.782373:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:2.0:1784948793.782373:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000200:1.0:1784948793.782376:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948793.782379:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948793.782381:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948793.782393:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948793.782393:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948793.782395:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000010aeeaab. 00000100:00000001:0.0:1784948793.782402:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948793.782402:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948793.782403:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f29bca31. 00000100:00001000:2.0:1784948793.782415:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=115 v=5 (5 5 5 5) 00000100:00000001:0.0:1784948793.782417:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948793.782423:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00001000:2.0:1784948793.782426:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=115 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948793.782428:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948793.782439:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948793.782442:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948793.782446:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784948793.782460:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948793.782461:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948793.782465:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948793.782474:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948793.782480:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948793.782486:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948793.782494:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784948793.782495:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 681 00000100:00000040:0.0:1784948793.782501:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948793.782504:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948793.782505:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470014336 : -115544239537280 : ffff96e9c2a58380) 00000100:00000001:2.0:1784948793.782510:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948793.782516:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000049907b7f x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:454/0 lens 224/0 e 0 to 0 dl 1784948804 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948793.782523:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948793.782540:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948793.782545:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784948793.782551:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784948793.782552:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fa1e143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289652864:0@lo:400: 00000020:00000010:0.0:1784948793.782554:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00100000:0.0:1784948793.782564:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000049907b7f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289653120:12345-192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948793.782566:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948793.782569:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289653120 00000100:00000001:2.0:1784948793.782595:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784948793.782597:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948793.782603:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:2.0:1784948793.782606:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fa1e143a x1871651289652864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948793.782607:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948793.782613:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948793.782617:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948793.782623:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948793.782627:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948793.782636:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784948793.782637:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784948793.782639:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948793.782645:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948793.782646:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:0.0:1784948793.782648:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 02000000:00000010:2.0:1784948793.782651:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000f1904ca. 00000100:00000001:0.0:1784948793.782654:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948793.782658:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:2.0:1784948793.782690:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:0.0:1784948793.782708:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f0f553eb. 00000020:00000001:2.0:1784948793.782710:0:337122:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:0.0:1784948793.782714:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948793.782715:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948793.782720:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948793.782722:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:0.0:1784948793.782724:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948793.782728:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 00010000:00000040:0.0:1784948793.782729:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289653120 00010000:00000001:0.0:1784948793.782736:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:2.0:1784948793.782738:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00010000:00000200:0.0:1784948793.782742:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000049907b7f x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:454/0 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948793.782744:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948793.782749:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fa1e143a. 00000100:00000001:2.0:1784948793.782758:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948793.782763:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948793.782765:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784948793.782767:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948793.782771:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948793.782776:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00001000:0.0:1784948793.782778:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=110 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948793.782782:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948793.782788:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948793.782789:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948793.782794:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948793.782797:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948793.782804:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948793.782808:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948793.782811:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948793.782815:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948793.782820:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948793.782825:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002e7a7cb2. 00000100:00000200:0.0:1784948793.782830:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289653120, offset 224 00000400:00000200:0.0:1784948793.782838:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948793.782848:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948793.782855:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3231:3231:250:4294967295] 192.168.202.137@tcp LPNI seq info [2104:2104:8:4294967295] 00000400:00000200:0.0:1784948793.782869:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948793.782876:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948793.782882:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000800:00000200:0.0:1784948793.782888:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948793.782895:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948793.782899:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948793.782939:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948793.782943:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948793.782946:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948793.782948:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948793.782950:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948793.782955:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000049907b7f x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:454/0 lens 224/224 e 0 to 0 dl 1784948804 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948793.782964:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000049907b7f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289653120:12345-192.168.202.137@tcp:400: Request processed in 424us (864us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948793.782975:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 681 00000100:00000040:0.0:1784948793.782978:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948793.782980:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948793.782982:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948793.782985:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007035a559. 00000020:00000010:0.0:1784948793.782989:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948793.782993:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 00000020:00000040:0.0:1784948793.782997:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948793.782999:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948793.783089:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948793.783095:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000400:00000200:2.0:1784948793.783100:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948793.783107:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948793.783113:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e7a7cb2 00000400:00000010:2.0:1784948793.783116:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002e7a7cb2. 00000100:00000001:2.0:1784948793.783120:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948793.783123:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948793.783132:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948793.783139:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948793.783143:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948793.783147:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948793.783156:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948793.783168:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6435f80 00000400:00000200:2.0:1784948793.783176:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8321 [1] + 224 00000800:00000001:2.0:1784948793.783183:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948793.783191:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948793.783194:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948793.783199:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948793.783205:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948793.783210:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948793.783223:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948793.783248:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948793.783255:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948793.783259:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.783275:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948793.783280:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948793.783286:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948793.783288:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948793.783290:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e8e1fecb. 00000400:00000200:3.0:1784948793.783297:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948793.783299:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000001:3.0:1784948793.783302:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948793.783308:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948793.783317:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948793.783324:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948793.783326:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.783328:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948793.783329:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948793.783331:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948793.783333:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948793.783334:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948793.783339:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=110 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948793.783342:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=110 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948793.783344:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948793.783345:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948793.783347:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948793.783350:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948793.783357:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.783360:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948793.783361:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948793.783362:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948793.783364:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948793.783366:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948793.783373:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948793.783386:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948793.783387:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948793.783389:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948793.783396:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000092889a66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289653120:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948793.783400:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.783402:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948793.783404:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000092889a66 x1871651289653120/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948809 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948793.783410:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948793.783411:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948793.783413:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a7af9ab9. 02000000:00000001:3.0:1784948793.783417:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948793.783418:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948793.783419:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948793.783421:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948793.783422:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:3.0:1784948793.783425:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948793.783426:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948793.783427:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000092889a66. 00000100:00000001:3.0:1784948793.783430:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948793.783431:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.783432:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948793.783436:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948793.783437:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948793.783439:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948793.783441:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948793.823988:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305733163, at 4305733164 00000100:00000010:3.0:1784948793.824000:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fd993a25. 00000100:00000040:0.0:1784948793.824017:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305733163, at 4305733164 00000100:00000040:2.0:1784948793.824029:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305733163, at 4305733164 00000100:00000010:0.0:1784948793.824030:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00000100:00000040:1.0:1784948793.824033:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305733163, at 4305733164 00000100:00000010:2.0:1784948793.824044:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000100:00000010:1.0:1784948793.824049:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000445487c0. 00000800:00000100:2.0:1784948793.972086:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000800:00000010:2.0:1784948793.972096:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e6aac0b. 00000400:00000200:2.0:1784948793.972110:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948793.972120:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948793.972132:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11067 health :900 00000400:00000200:2.0:1784948793.972140:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948793.972145:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:2.0:1784948793.972152:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000800:00000100:2.0:1784948793.972190:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.25@tcp 00000800:00000010:2.0:1784948793.972195:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003547cd5f. 00000400:00000200:2.0:1784948793.972203:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948793.972211:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.25@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948793.972218:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948793.972221:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.972233:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000200:2.0:1784948793.972236:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.25@tcp: 1 00000400:00000001:1.0:1784948793.972247:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1784948793.972249:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.25@tcp: -114 00000400:00000001:1.0:1784948793.972252:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.972259:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000400:00000010:1.0:1784948793.972262:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000400:00000200:1.0:1784948793.972268:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.972276:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:1.0:1784948793.972282:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004b465f54. 00000400:00000200:1.0:1784948793.972291:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:1.0:1784948793.972296:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000400:00000200:1.0:1784948793.972301:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00000800:00000100:2.0:1784948793.972310:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000400:00000200:1.0:1784948793.972321:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.25@tcp(000000003b287909) state 0x840 00000800:00000010:2.0:1784948793.972322:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d9530b6c. 00000400:00000001:1.0:1784948793.972329:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.972333:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948793.972334:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948793.972336:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000400:00000010:1.0:1784948793.972342:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000400:00000200:1.0:1784948793.972349:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948793.972355:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948793.972361:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.25@tcp: -114 00000400:00000010:1.0:1784948793.972368:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000011003cc7. 00000400:00000200:1.0:1784948793.972373:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.25@tcp:-110 00000400:00000200:1.0:1784948793.972377:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.25@tcp(000000003b287909) state 0x2040 rc -110 00000400:00000200:2.0:1784948793.972377:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.972388:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.25@tcp 00000400:00000200:2.0:1784948793.972404:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:2.0:1784948793.972419:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:1.0:1784948793.972434:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:1.0:1784948793.972440:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.972442:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.972444:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000400:00000010:1.0:1784948793.972447:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000400:00000200:1.0:1784948793.972452:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.972456:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:1.0:1784948793.972459:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000013f68999. 00000400:00000200:1.0:1784948793.972464:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:1.0:1784948793.972477:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:1.0:1784948793.972480:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000400:00000200:3.0:1784948793.972532:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.25@tcp NID 111.222.142.25@tcp: -110. discovery complete 00000400:00000200:3.0:1784948793.972542:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.142.25@tcp primary NID 111.222.142.25@tcp rc -110 00000100:00000010:3.0:1784948793.972559:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 000000003f90f5f2. 00000100:00000001:3.0:1784948793.972590:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784948793.972594:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=000000003f90f5f2 refcount 1 to 111.222.142.25@tcp 00010000:00000010:3.0:1784948793.972599:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000fd993a25. 00010000:00080000:3.0:1784948793.972686:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.142.25@tcp at tail 00010000:00000001:3.0:1784948793.972690:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948793.972694:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948793.972708:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 000000005b888794. 00000100:00000400:3.0:1784948793.972714:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.173.26@tcp: rc = -75 00000100:00000040:3.0:1784948793.972718:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.173.26@tcp cnt=32 00000400:00000200:3.0:1784948793.972721:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784948793.972725:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784948793.972728:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784948793.972731:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784948793.972733:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784948793.972740:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784948793.972746:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.972750:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.972753:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784948793.972755:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784948793.972759:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.972780:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784948793.972784:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.972785:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.972788:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.972791:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784948793.972793:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784948793.972796:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.972800:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784948793.972803:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.972805:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.972808:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.972812:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784948793.972814:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784948793.972818:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.972822:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784948793.972825:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.972832:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.972833:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948793.972842:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.972846:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784948793.972848:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:3.0:1784948793.972852:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.972858:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784948793.972866:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:3.0:1784948793.972870:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.972876:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.972879:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000010:1.0:1784948793.972880:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000200:3.0:1784948793.972882:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784948793.972891:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000010:1.0:1784948793.972892:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000400:00000200:3.0:1784948793.972897:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.972943:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784948793.972945:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.972950:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:1.0:1784948793.972951:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000022dbcdce. 00000400:00000200:3.0:1784948793.972963:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.972970:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:1.0:1784948793.972971:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784948793.972979:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000200:3.0:1784948793.972987:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.972997:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:1.0:1784948793.973004:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.973009:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.973013:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.973017:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:1.0:1784948793.973021:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3232:3232:253:4294967295] 111.222.173.1@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:1.0:1784948793.973029:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:1.0:1784948793.973037:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.973045:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784948793.973051:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000800:00000010:1.0:1784948793.973052:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002a753d9d. 00000400:00000200:3.0:1784948793.973055:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973061:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784948793.973071:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784948793.973071:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973084:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:3.0:1784948793.973090:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000800:00000200:1.1:1784948793.973092:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784948793.973100:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.973109:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784948793.973111:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973113:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.973115:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973117:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784948793.973120:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:3.0:1784948793.973129:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973130:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:1.0:1784948793.973145:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:3.0:1784948793.973146:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:3.0:1784948793.973154:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973162:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.973168:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784948793.973172:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973180:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784948793.973183:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000010:1.0:1784948793.973184:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000010:1.0:1784948793.973193:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000400:00000200:3.0:1784948793.973194:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.973206:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:3.0:1784948793.973209:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784948793.973213:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000851e368. 00000400:00000200:3.0:1784948793.973214:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.973225:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:1.0:1784948793.973234:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784948793.973235:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784948793.973243:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:1.0:1784948793.973250:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784948793.973252:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973265:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.973270:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.973273:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.973276:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:1.0:1784948793.973280:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3233:3233:252:4294967295] 111.222.142.1@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:1.0:1784948793.973286:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:1.0:1784948793.973292:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.973299:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784948793.973305:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000800:00000010:1.0:1784948793.973306:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000635133da. 00000400:00000200:3.0:1784948793.973311:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973317:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784948793.973324:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784948793.973333:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000800:00000200:1.0:1784948793.973335:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 13 secs later) 00000400:00000200:3.0:1784948793.973339:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000200:3.0:1784948793.973349:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000800:00000200:1.1:1784948793.973357:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784948793.973362:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000800:00000200:1.1:1784948793.973365:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 13 secs later) 00000400:00000200:3.0:1784948793.973374:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:3.0:1784948793.973382:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000100:1.0:1784948793.973383:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784948793.973390:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000010:1.0:1784948793.973391:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000635133da. 00000400:00000200:1.0:1784948793.973402:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948793.973403:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973414:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:1.0:1784948793.973422:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948793.973424:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:1.0:1784948793.973432:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948793.973434:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973443:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:3.0:1784948793.973448:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784948793.973453:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948793.973566:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:3.0:1784948793.973574:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.973580:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973583:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:3.0:1784948793.973585:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:3.0:1784948793.973591:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973595:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:3.0:1784948793.973604:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:1.0:1784948793.973606:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 rc 0 00000400:00000200:3.0:1784948793.973613:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973621:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.973626:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784948793.973635:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973642:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:3.0:1784948793.973644:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000010:1.0:1784948793.973646:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000013f68999. 00000400:00000200:3.0:1784948793.973648:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784948793.973660:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000200:3.0:1784948793.973677:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784948793.973681:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973683:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.973686:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973696:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000010:1.0:1784948793.973700:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009f8441cb. 00000400:00000200:3.0:1784948793.973701:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:3.0:1784948793.973712:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973719:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784948793.973727:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784948793.973730:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973735:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.973738:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973740:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:3.0:1784948793.973743:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:1.0:1784948793.973750:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:3.0:1784948793.973751:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973763:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.973769:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.973772:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.973775:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:1.0:1784948793.973778:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3234:3234:252:4294967295] 111.222.173.2@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:1.0:1784948793.973783:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:1.0:1784948793.973789:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.973797:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784948793.973803:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000800:00000010:1.0:1784948793.973804:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000635133da. 00000400:00000200:3.0:1784948793.973809:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973816:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784948793.973822:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784948793.973826:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973833:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000800:00000200:1.1:1784948793.973841:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784948793.973842:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:3.0:1784948793.973856:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973859:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:1.0:1784948793.973867:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:3.0:1784948793.973873:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784948793.973877:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973882:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.973888:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:3.0:1784948793.973891:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973901:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784948793.973926:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:3.0:1784948793.973933:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.973939:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000010:1.0:1784948793.973940:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000011003cc7. 00000400:00000200:3.0:1784948793.973942:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.973951:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:1.0:1784948793.973952:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000400:00000200:3.0:1784948793.973958:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.973964:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000010:1.0:1784948793.973966:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e3180272. 00000400:00000200:3.0:1784948793.973968:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:3.0:1784948793.973985:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.973989:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784948793.973995:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:3.0:1784948793.974000:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974005:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.974008:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974011:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784948793.974014:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:1.0:1784948793.974019:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:3.0:1784948793.974023:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.974029:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.974036:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.974038:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.974041:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:1.0:1784948793.974044:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3235:3235:251:4294967295] 111.222.142.2@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:1.0:1784948793.974049:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:1.0:1784948793.974055:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.974063:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000800:00000010:1.0:1784948793.974067:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008f5595bf. 00000400:00000200:3.0:1784948793.974068:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:3.0:1784948793.974075:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974078:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784948793.974085:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784948793.974092:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974099:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000800:00000200:1.1:1784948793.974101:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784948793.974107:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:1.0:1784948793.974115:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:3.0:1784948793.974120:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.974128:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x261 rc 0 00000400:00000200:3.0:1784948793.974135:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:3.0:1784948793.974140:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974144:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.974149:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784948793.974157:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974163:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000010:1.0:1784948793.974167:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000053ab16eb. 00000400:00000200:3.0:1784948793.974167:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000010:1.0:1784948793.974177:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000400:00000200:3.0:1784948793.974178:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784948793.974186:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d3823a3e. 00000400:00000200:3.0:1784948793.974192:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784948793.974196:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784948793.974203:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784948793.974205:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.974212:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974216:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:3.0:1784948793.974218:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:1.0:1784948793.974224:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:3.0:1784948793.974226:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.974236:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.974241:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.974243:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.974246:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:1.0:1784948793.974255:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3236:3236:250:4294967295] 111.222.173.3@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:1.0:1784948793.974260:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:1.0:1784948793.974266:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.974272:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:1.0:1784948793.974275:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003a8a98bc. 00000400:00000200:3.0:1784948793.974277:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784948793.974284:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974287:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784948793.974291:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784948793.974298:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974304:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000800:00000200:1.1:1784948793.974310:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784948793.974314:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:3.0:1784948793.974322:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.974327:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:3.0:1784948793.974329:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974331:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.974336:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:1.0:1784948793.974342:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784948793.974344:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:1.0:1784948793.974350:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:3.0:1784948793.974357:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:1.0:1784948793.974365:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:3.0:1784948793.974372:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784948793.974387:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fff5e130. 00000400:00000200:3.0:1784948793.974392:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000010:1.0:1784948793.974396:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b039801a. 00000400:00000200:3.0:1784948793.974398:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974408:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.974414:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974417:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000010:1.0:1784948793.974418:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f9fb66e4. 00000400:00000200:3.0:1784948793.974420:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000200:1.0:1784948793.974432:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784948793.974433:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.974454:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:1.0:1784948793.974466:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.974469:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.974472:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.974476:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:1.0:1784948793.974481:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3237:3237:249:4294967295] 111.222.142.3@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:1.0:1784948793.974488:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:1.0:1784948793.974501:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.974514:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784948793.974517:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784948793.974522:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974524:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948793.974527:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974535:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:3.0:1784948793.974538:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000800:00000010:1.0:1784948793.974539:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000353a5c12. 00000400:00000200:3.0:1784948793.974545:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948793.974562:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000800:00000200:1.0:1784948793.974566:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784948793.974567:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974575:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784948793.974577:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 13 secs later) 00000400:00000200:3.0:1784948793.974587:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974592:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000800:00000200:1.1:1784948793.974600:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784948793.974601:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000800:00000200:1.1:1784948793.974610:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 13 secs later) 00000400:00000200:3.0:1784948793.974617:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000800:00000100:1.0:1784948793.974633:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784948793.974633:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000800:00000010:1.0:1784948793.974641:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000353a5c12. 00000400:00000200:3.0:1784948793.974642:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974652:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.974653:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948793.974660:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974691:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:1.0:1784948793.974696:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948793.974703:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:1.0:1784948793.974706:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948793.974717:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784948793.974718:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:1.0:1784948793.974730:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000200:1.0:1784948793.974743:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:1.0:1784948793.974747:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 rc 0 00000400:00000200:3.0:1784948793.974754:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784948793.974761:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948793.974767:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.974773:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784948793.974784:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974790:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784948793.974793:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000010:1.0:1784948793.974794:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000461737cb. 00000400:00000200:3.0:1784948793.974803:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784948793.974804:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000400:00000200:3.0:1784948793.974822:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000200:3.0:1784948793.974828:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784948793.974958:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000007e2f3d0. 00000400:00000200:3.0:1784948793.974960:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784948793.974975:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784948793.974976:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948793.974986:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000100:00000010:3.0:1784948793.974991:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 000000005b888794. 00000400:00000200:1.0:1784948793.974996:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000020:00000001:3.0:1784948793.975006:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000200:1.0:1784948793.975008:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1784948793.975014:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000400:00000200:1.0:1784948793.975016:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948793.975024:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:1.0:1784948793.975025:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.975033:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:1.0:1784948793.975036:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3238:3238:249:4294967295] 111.222.173.4@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:1.0:1784948793.975041:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:1.0:1784948793.975047:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.975053:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000400:00000010:3.0:1784948793.975058:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000a7af9ab9. 00000800:00000010:1.0:1784948793.975059:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000353a5c12. 00000400:00000200:3.0:1784948793.975072:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000a7af9ab9 nid 111.222.173.26@tcp 00000800:00000200:1.0:1784948793.975075:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000010:3.0:1784948793.975081:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000000afdabce. 00000800:00000200:1.1:1784948793.975090:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784948793.975093:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 000000000afdabce net tcp 00000400:00000010:3.0:1784948793.975102:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000005e186afc. 00000400:00000200:1.0:1784948793.975103:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:3.0:1784948793.975117:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 000000005e186afc nid 111.222.173.26@tcp 00000400:00000200:1.0:1784948793.975118:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784948793.975135:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.173.26@tcp NID 111.222.173.26@tcp flags 0x0 00000400:00000200:3.0:1784948793.975141:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.173.26@tcp: 0 00000400:00000200:1.0:1784948793.975147:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000200:3.0:1784948793.975155:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.26@tcp: 0 00000400:00000200:3.0:1784948793.975159:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784948793.975169:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006b8bfa56. 00000400:00000010:1.0:1784948793.975178:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000400:00000010:1.0:1784948793.975186:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a3f431e3. 00000400:00000200:1.0:1784948793.975194:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:1.0:1784948793.975202:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:1.0:1784948793.975208:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.975212:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.975217:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.975222:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:1.0:1784948793.975230:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3239:3239:248:4294967295] 111.222.173.5@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:1.0:1784948793.975237:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:1.0:1784948793.975253:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.975257:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000800:00000010:1.0:1784948793.975262:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000062b4a223. 00000800:00000200:1.0:1784948793.975269:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784948793.975273:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:1.0:1784948793.975292:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:1.0:1784948793.975296:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x261 rc 0 00000400:00000200:1.0:1784948793.975301:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:1.0:1784948793.975307:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000eae26e4. 00000400:00000010:1.0:1784948793.975311:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000010:1.0:1784948793.975321:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000014336a14. 00000400:00000200:1.0:1784948793.975325:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:1.0:1784948793.975330:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:1.0:1784948793.975333:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.975337:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.975343:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.975348:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:1.0:1784948793.975352:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3240:3240:247:4294967295] 111.222.142.5@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:1.0:1784948793.975358:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:1.0:1784948793.975368:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.975374:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:1.0:1784948793.975385:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d104522b. 00000800:00000200:1.0:1784948793.975389:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.0:1784948793.975392:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784948793.975397:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784948793.975400:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 13 secs later) 00000800:00000100:1.0:1784948793.975404:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.5@tcp 00000800:00000010:1.0:1784948793.975406:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d104522b. 00000400:00000200:1.0:1784948793.975410:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948793.975414:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948793.975418:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.975422:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:1.0:1784948793.975425:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:1.0:1784948793.975432:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:1.0:1784948793.975434:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 rc 0 00000400:00000200:1.0:1784948793.975437:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:1.0:1784948793.975443:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000684a4a3c. 00000400:00000010:1.0:1784948793.975447:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000400:00000010:1.0:1784948793.975461:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e3882e96. 00000400:00000200:1.0:1784948793.975466:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:1.0:1784948793.975472:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:1.0:1784948793.975476:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.975482:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.975486:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.975490:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:1.0:1784948793.975496:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3241:3241:247:4294967295] 111.222.142.6@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:1.0:1784948793.975503:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:1.0:1784948793.975513:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.975523:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:1.0:1784948793.975529:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d104522b. 00000800:00000200:1.0:1784948793.975536:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784948793.975544:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000200:1.0:1784948793.975549:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:1.0:1784948793.975697:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x261 rc 0 00000400:00000200:1.0:1784948793.975704:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:1.0:1784948793.975708:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.975710:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.975711:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784948793.975713:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000400:00000200:1.0:1784948793.975717:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.975721:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:1.0:1784948793.975724:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000400:00000200:1.0:1784948793.975728:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-5 00000400:00000200:1.0:1784948793.975731:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.975735:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:1.0:1784948793.975738:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:1.0:1784948793.975744:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000010:1.0:1784948793.975747:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000400:00000010:1.0:1784948793.975752:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009ec02d9a. 00000400:00000200:1.0:1784948793.975755:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:1.0:1784948793.975761:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:1.0:1784948793.975765:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.975768:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.975771:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.975774:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:1.0:1784948793.975777:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3242:3242:246:4294967295] 111.222.173.7@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:1.0:1784948793.975783:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:1.0:1784948793.975790:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.975795:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:1.0:1784948793.975799:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000057c7ab4. 00000800:00000200:1.0:1784948793.975804:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784948793.975808:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:1.0:1784948793.975829:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:1.0:1784948793.975831:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000400:00000200:1.0:1784948793.975835:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:1.0:1784948793.975839:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000010:1.0:1784948793.975842:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000010:1.0:1784948793.975846:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000cef8b3cd. 00000400:00000200:1.0:1784948793.975850:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:1.0:1784948793.975854:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:1.0:1784948793.975857:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.975860:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.975862:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.975865:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:1.0:1784948793.975870:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3243:3243:245:4294967295] 111.222.142.7@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:1.0:1784948793.975876:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:1.0:1784948793.975884:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.975889:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:1.0:1784948793.975892:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000820f02b3. 00000800:00000200:1.0:1784948793.975897:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.0:1784948793.975900:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784948793.975941:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784948793.975946:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 13 secs later) 00000800:00000100:1.0:1784948793.975959:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.7@tcp 00000800:00000010:1.0:1784948793.975962:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000820f02b3. 00000400:00000200:1.0:1784948793.975968:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948793.975975:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948793.975979:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.975983:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:1.0:1784948793.975990:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:1.0:1784948793.976002:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:1.0:1784948793.976007:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 rc 0 00000400:00000200:1.0:1784948793.976015:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:1.0:1784948793.976022:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004041a48e. 00000400:00000010:1.0:1784948793.976026:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000010:1.0:1784948793.976031:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a8bf092e. 00000400:00000200:1.0:1784948793.976035:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:1.0:1784948793.976041:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:1.0:1784948793.976044:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.976047:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.976050:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.976053:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:1.0:1784948793.976057:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3244:3244:245:4294967295] 111.222.173.8@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:1.0:1784948793.976062:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:1.0:1784948793.976070:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.976075:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:1.0:1784948793.976080:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000820f02b3. 00000800:00000200:1.0:1784948793.976084:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784948793.976088:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:1.0:1784948793.976092:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:1.0:1784948793.976096:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:1.0:1784948793.976103:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:1.0:1784948793.976108:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b105ef1a. 00000400:00000010:1.0:1784948793.976112:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 00000400:00000010:1.0:1784948793.976116:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000339be500. 00000400:00000200:1.0:1784948793.976119:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:1.0:1784948793.976125:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:1.0:1784948793.976131:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.976134:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.976136:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.976139:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:1.0:1784948793.976143:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3245:3245:244:4294967295] 111.222.142.8@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:1.0:1784948793.976148:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:1.0:1784948793.976154:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.976158:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:1.0:1784948793.976161:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009ff40a64. 00000800:00000200:1.0:1784948793.976165:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784948793.976169:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000400:00000200:1.0:1784948793.976172:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:1.0:1784948793.976174:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x261 rc 0 00000400:00000200:1.0:1784948793.976177:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:1.0:1784948793.976180:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000038ac69a2. 00000400:00000010:1.0:1784948793.976182:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000010:1.0:1784948793.976185:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000171e706a. 00000400:00000200:1.0:1784948793.976188:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:1.0:1784948793.976192:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:1.0:1784948793.976253:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.976258:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.976261:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.976263:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:1.0:1784948793.976266:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3246:3246:243:4294967295] 111.222.173.9@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:1.0:1784948793.976271:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:1.0:1784948793.976277:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.976283:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:1.0:1784948793.976287:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005a1c05a3. 00000800:00000200:1.0:1784948793.976292:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784948793.976296:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000400:00000200:1.0:1784948793.976313:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:1.0:1784948793.976315:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x261 rc 0 00000400:00000200:1.0:1784948793.976319:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:1.0:1784948793.976324:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006620b0a1. 00000400:00000010:1.0:1784948793.976327:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000400:00000010:1.0:1784948793.976342:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006d2c4b24. 00000400:00000200:1.0:1784948793.976347:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:1.0:1784948793.976355:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:1.0:1784948793.976360:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.976363:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.976366:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.976370:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:1.0:1784948793.976375:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3247:3247:242:4294967295] 111.222.142.9@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:1.0:1784948793.976380:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:1.0:1784948793.976386:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.976391:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:1.0:1784948793.976407:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000034aacccc. 00000800:00000200:1.0:1784948793.976414:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.0:1784948793.976417:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784948793.976422:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784948793.976445:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 13 secs later) 00000800:00000100:1.0:1784948793.976454:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.9@tcp 00000800:00000010:1.0:1784948793.976457:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000034aacccc. 00000400:00000200:1.0:1784948793.976464:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948793.976472:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948793.976479:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.976484:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:1.0:1784948793.976490:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000200:1.0:1784948793.976504:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:1.0:1784948793.976510:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 rc 0 00000400:00000200:1.0:1784948793.976518:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:1.0:1784948793.976532:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000898cc47a. 00000400:00000010:1.0:1784948793.976539:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007d821c01. 00000400:00000010:1.0:1784948793.976546:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000091a157d3. 00000400:00000200:1.0:1784948793.976552:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:1.0:1784948793.976562:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:1.0:1784948793.976567:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.976572:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.976576:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.976581:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:1.0:1784948793.976587:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3248:3248:242:4294967295] 111.222.142.10@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:1.0:1784948793.976597:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:1.0:1784948793.976607:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.976616:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:1.0:1784948793.976622:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000034aacccc. 00000800:00000200:1.0:1784948793.976739:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784948793.976743:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000400:00000200:1.0:1784948793.976748:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:1.0:1784948793.976750:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x261 rc 0 00000400:00000200:1.0:1784948793.976754:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:1.0:1784948793.976760:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009802f36f. 00000400:00000010:1.0:1784948793.976764:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000400:00000010:1.0:1784948793.976768:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003b463cff. 00000400:00000200:1.0:1784948793.976772:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:1.0:1784948793.976778:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:1.0:1784948793.976782:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.976785:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.976787:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.976790:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:1.0:1784948793.976794:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3249:3249:241:4294967295] 111.222.173.11@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:1.0:1784948793.976798:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:1.0:1784948793.976805:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.976810:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:1.0:1784948793.976813:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000036e798e7. 00000800:00000200:1.0:1784948793.976817:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784948793.976821:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:1.0:1784948793.976839:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:1.0:1784948793.976842:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:1.0:1784948793.976846:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:1.0:1784948793.976850:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e120de1c. 00000400:00000010:1.0:1784948793.976853:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000010:1.0:1784948793.976856:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f65f2fb7. 00000400:00000200:1.0:1784948793.976860:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:1.0:1784948793.976865:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:1.0:1784948793.976867:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.976870:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.976873:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.976876:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:1.0:1784948793.976881:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3250:3250:240:4294967295] 111.222.142.11@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:1.0:1784948793.976887:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:1.0:1784948793.976894:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.976900:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:1.0:1784948793.976944:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000662152ce. 00000800:00000200:1.0:1784948793.976951:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.0:1784948793.976955:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 13 secs later) 00000800:00000200:1.1:1784948793.976962:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784948793.976966:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 13 secs later) 00000800:00000100:1.0:1784948793.976973:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.11@tcp 00000800:00000010:1.0:1784948793.976977:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000662152ce. 00000400:00000200:1.0:1784948793.976981:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948793.976987:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948793.976992:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.976998:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:1.0:1784948793.977003:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:1.0:1784948793.977011:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:1.0:1784948793.977014:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 rc 0 00000400:00000200:1.0:1784948793.977019:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:1.0:1784948793.977025:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eb983bef. 00000400:00000010:1.0:1784948793.977030:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000010:1.0:1784948793.977034:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000041ec6752. 00000400:00000200:1.0:1784948793.977038:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:1.0:1784948793.977044:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:1.0:1784948793.977047:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977051:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977053:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977056:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:1.0:1784948793.977060:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3251:3251:240:4294967295] 111.222.173.12@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:1.0:1784948793.977065:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:1.0:1784948793.977077:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977083:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:1.0:1784948793.977087:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000662152ce. 00000800:00000200:1.0:1784948793.977092:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784948793.977098:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:1.0:1784948793.977102:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:1.0:1784948793.977105:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:1.0:1784948793.977108:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:1.0:1784948793.977113:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000043c3e1d8. 00000400:00000010:1.0:1784948793.977115:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000400:00000010:1.0:1784948793.977119:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009e0071c7. 00000400:00000200:1.0:1784948793.977122:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:1.0:1784948793.977126:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:1.0:1784948793.977128:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977131:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977133:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977136:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:1.0:1784948793.977139:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3252:3252:239:4294967295] 111.222.142.12@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:1.0:1784948793.977144:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:1.0:1784948793.977150:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977154:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:1.0:1784948793.977157:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003ed79484. 00000800:00000200:1.0:1784948793.977160:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784948793.977164:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:1.0:1784948793.977167:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:1.0:1784948793.977169:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:1.0:1784948793.977172:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:1.0:1784948793.977175:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000be26f34f. 00000400:00000010:1.0:1784948793.977177:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000010:1.0:1784948793.977181:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f12dd04d. 00000400:00000200:1.0:1784948793.977269:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:1.0:1784948793.977275:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:1.0:1784948793.977279:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977282:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977284:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977287:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:1.0:1784948793.977290:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3253:3253:238:4294967295] 111.222.173.13@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:1.0:1784948793.977296:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:1.0:1784948793.977302:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977307:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:1.0:1784948793.977312:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a328590b. 00000800:00000200:1.0:1784948793.977316:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784948793.977320:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:1.0:1784948793.977337:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:1.0:1784948793.977340:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:1.0:1784948793.977344:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:1.0:1784948793.977348:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006841197d. 00000400:00000010:1.0:1784948793.977351:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000400:00000010:1.0:1784948793.977356:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003faefc6b. 00000400:00000200:1.0:1784948793.977359:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:1.0:1784948793.977363:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:1.0:1784948793.977366:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977369:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977371:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977374:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:1.0:1784948793.977377:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3254:3254:237:4294967295] 111.222.142.13@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:1.0:1784948793.977382:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:1.0:1784948793.977388:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977392:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:1.0:1784948793.977396:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000800:00000200:1.0:1784948793.977400:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.0:1784948793.977403:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.13 (cnted 0, interval 60, 21 secs later) 00000800:00000200:1.1:1784948793.977408:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784948793.977410:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.13 (cnted 0, interval 60, 21 secs later) 00000800:00000100:1.0:1784948793.977415:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.13@tcp 00000800:00000010:1.0:1784948793.977417:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000400:00000200:1.0:1784948793.977422:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948793.977426:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948793.977430:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.977436:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:1.0:1784948793.977440:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:1.0:1784948793.977446:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:1.0:1784948793.977448:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 rc 0 00000400:00000200:1.0:1784948793.977452:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:1.0:1784948793.977462:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000047e19e91. 00000400:00000010:1.0:1784948793.977467:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000400:00000010:1.0:1784948793.977473:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000afa63392. 00000400:00000200:1.0:1784948793.977481:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:1.0:1784948793.977489:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:1.0:1784948793.977494:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977499:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977504:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977509:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:1.0:1784948793.977515:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3255:3255:237:4294967295] 111.222.173.14@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:1.0:1784948793.977522:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:1.0:1784948793.977539:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977549:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:1.0:1784948793.977557:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000800:00000200:1.0:1784948793.977564:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:1.1:1784948793.977571:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:1.0:1784948793.977578:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:1.0:1784948793.977581:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:1.0:1784948793.977587:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:1.0:1784948793.977593:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002c1a3d41. 00000400:00000010:1.0:1784948793.977598:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000400:00000010:1.0:1784948793.977605:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003859b1c5. 00000400:00000200:1.0:1784948793.977701:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:1.0:1784948793.977710:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:1.0:1784948793.977713:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977716:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977720:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977723:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:1.0:1784948793.977726:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3256:3256:236:4294967295] 111.222.142.14@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:1.0:1784948793.977731:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:1.0:1784948793.977738:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977743:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:1.0:1784948793.977748:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c2b2d4f6. 00000800:00000200:1.0:1784948793.977753:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784948793.977757:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:1.0:1784948793.977761:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:1.0:1784948793.977763:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:1.0:1784948793.977767:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 00000400:00000001:1.0:1784948793.977770:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.977772:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.977774:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b039801a 00000400:00000010:1.0:1784948793.977776:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b039801a. 00000400:00000200:1.0:1784948793.977780:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.977783:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:1.0:1784948793.977787:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fff5e130. 00000400:00000200:1.0:1784948793.977791:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-5 00000400:00000200:1.0:1784948793.977794:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.977797:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00000400:00000200:1.0:1784948793.977801:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:1.0:1784948793.977805:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fff5e130. 00000400:00000010:1.0:1784948793.977808:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b039801a. 00000400:00000010:1.0:1784948793.977811:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000548ea7b. 00000400:00000200:1.0:1784948793.977815:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:1.0:1784948793.977818:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:1.0:1784948793.977821:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977825:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977827:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977830:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:1.0:1784948793.977834:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3257:3257:235:4294967295] 111.222.173.15@tcp LPNI seq info [21:21:8:4294967295] 00000400:00000200:1.0:1784948793.977838:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:1.0:1784948793.977844:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977848:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:1.0:1784948793.977851:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000800:00000200:1.0:1784948793.977855:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784948793.977858:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:1.0:1784948793.977876:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:1.0:1784948793.977878:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:1.0:1784948793.977882:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:1.0:1784948793.977886:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000da755428. 00000400:00000010:1.0:1784948793.977888:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000010:1.0:1784948793.977892:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c241e266. 00000400:00000200:1.0:1784948793.977895:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:1.0:1784948793.977901:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:1.0:1784948793.977936:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948793.977939:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.977942:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948793.977947:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:1.0:1784948793.977953:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3258:3258:234:4294967295] 111.222.142.15@tcp LPNI seq info [20:20:8:4294967295] 00000400:00000200:1.0:1784948793.977958:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:1.0:1784948793.977966:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.977975:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:1.0:1784948793.977981:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000800:00000200:1.0:1784948793.977989:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.0:1784948793.977992:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.15 (cnted 0, interval 60, 21 secs later) 00000800:00000200:1.1:1784948793.977998:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784948793.978000:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.15 (cnted 0, interval 60, 21 secs later) 00000800:00000100:1.0:1784948793.978005:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.15@tcp 00000800:00000010:1.0:1784948793.978007:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:1.0:1784948793.978012:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948793.978017:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948793.978021:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948793.978023:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:1.0:1784948793.978026:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:1.0:1784948793.978038:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:1.0:1784948793.978041:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 rc 0 00000400:00000200:1.0:1784948793.978045:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.26@tcp(00000000a7af9ab9) state 0x40 00000400:00000010:1.0:1784948793.978070:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf393d23. 00000400:00000010:1.0:1784948793.978076:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000400:00000010:1.0:1784948793.978081:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a705132d. 00000400:00000200:1.0:1784948793.978085:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.26@tcp 00000400:00000200:1.0:1784948793.978092:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.173.26@tcp local destination 00000400:00000200:1.0:1784948793.978096:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948793.978100:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.173.26@tcp 00000400:00000200:1.0:1784948793.978104:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.173.26@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:1.0:1784948793.978108:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3259:3259:234:4294967295] 111.222.173.26@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:1.0:1784948793.978113:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.26@tcp 00000400:00000200:1.0:1784948793.978119:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.26@tcp(111.222.173.26@tcp:111.222.173.26@tcp) : GET try# 0 00000800:00000200:1.0:1784948793.978124:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.26@tcp 00000800:00000010:1.0:1784948793.978126:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000800:00000010:1.0:1784948793.978133:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000db4ea83f. 00000800:00000010:1.0:1784948793.978137:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 000000002883c343. 00000800:00000200:1.0:1784948793.978142:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000db4ea83f] -> 12345-111.222.173.26@tcp (2) 00000800:00000200:1.1:1784948793.978145:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000db4ea83f] -> 12345-111.222.173.26@tcp (2) 00000400:00000200:1.0:1784948793.978165:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.26@tcp 00000400:00000200:1.0:1784948793.978167:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.26@tcp(00000000a7af9ab9) state 0x240 rc 0 00000400:00000200:1.0:1784948793.978171:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:1.0:1784948793.978174:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.978177:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.978179:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:1.0:1784948793.978180:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000400:00000200:1.0:1784948793.978184:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.978191:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:1.0:1784948793.978194:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000eae26e4. 00000400:00000200:1.0:1784948793.978201:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-5 00000400:00000200:1.0:1784948793.978203:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.978270:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000400:00000200:1.0:1784948793.978275:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000400:00000001:1.0:1784948793.978278:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.978280:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.978282:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:1.0:1784948793.978285:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:1.0:1784948793.978288:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.978291:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:1.0:1784948793.978295:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089e37819. 00000400:00000200:1.0:1784948793.978299:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-5 00000400:00000200:1.0:1784948793.978302:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.978305:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:1.0:1784948793.978308:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:1.0:1784948793.978311:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.978312:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.978314:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784948793.978315:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000400:00000200:1.0:1784948793.978319:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.978325:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:1.0:1784948793.978329:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006620b0a1. 00000400:00000200:1.0:1784948793.978333:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-5 00000400:00000200:1.0:1784948793.978335:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.978338:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:1.0:1784948793.978342:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:1.0:1784948793.978345:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.978348:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.978349:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784948793.978351:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000400:00000200:1.0:1784948793.978354:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.978357:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:1.0:1784948793.978360:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e120de1c. 00000400:00000200:1.0:1784948793.978364:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-5 00000400:00000200:1.0:1784948793.978366:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.978369:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000400:00000200:1.0:1784948793.978372:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:1.0:1784948793.978374:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.978375:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.978376:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000400:00000010:1.0:1784948793.978378:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000400:00000200:1.0:1784948793.978380:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.978382:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:1.0:1784948793.978384:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006841197d. 00000400:00000200:1.0:1784948793.978387:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-5 00000400:00000200:1.0:1784948793.978389:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.978392:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:1.0:1784948793.978395:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:1.0:1784948793.978397:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948793.978398:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948793.978399:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:1.0:1784948793.978401:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000400:00000200:1.0:1784948793.978403:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948793.978405:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:1.0:1784948793.978407:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000da755428. 00000400:00000200:1.0:1784948793.978411:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-5 00000400:00000200:1.0:1784948793.978413:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -5 00000400:00000200:1.0:1784948793.978416:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784948794.612027:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948794.612036:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948794.612045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948794.612047:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948794.612118:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948794.612125:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948794.612129:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948794.804013:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948794.804022:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948794.804039:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948793/real 0] req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948794 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.804059:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948794.804063:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948794.804066:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948794.804070:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948794 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.804081:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948794.804093:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948794.804097:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948794.804101:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948794.804104:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948794.804106:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948794.804110:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948794 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.804121:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784948794.804125:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948794.804129:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948794.804131:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948794.804134:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784948794.804138:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948794.804142:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784948794.804170:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784948794.804174:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784948794.804178:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948794 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948794.804188:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b13d185 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289652992:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784948794.804194:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948794.804198:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948794.804202:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b13d185 x1871651289652992/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948794 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.804212:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784948794.804217:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948794.804221:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784948794.804225:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948794.804230:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000900bf136. 02000000:00000001:0.0:1784948794.804238:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948794.804241:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948794.804244:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b13d185. 00000100:00000040:1.0:1784948794.804245:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948794.804251:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948794.804257:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.804259:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948794.804261:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948794.804268:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1784948794.804269:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b93d109. 00000100:00000001:0.0:1784948794.804270:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948794.804282:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948794.804284:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948794.804289:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.804294:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948794.804299:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948794.804306:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948794.804318:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000adf08f9b. 00000100:00000001:1.0:1784948794.804327:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.804337:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948794.804352:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948794.804355:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.804364:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005b93d109] to pc [ptlrpcd_00_01+1] req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948794.804411:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948794.804416:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948794.804421:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948794.804425:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948794.804428:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948794.804434:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784948794.804441:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11040 00000100:00000001:2.0:1784948794.804448:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948794.804455:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784948794.804457:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948794.804464:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1784948794.804470:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11065 00000100:00000001:0.0:1784948794.804472:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948794.804479:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.804481:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.804500:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948794.804504:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948794.804510:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948794.804514:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948794.804518:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948794.804523:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289653248:0@lo:400: 00000100:00000001:2.0:1784948794.804530:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948794.804536:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948794.804539:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948794.804542:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784948794.804548:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 02000000:00000001:2.0:1784948794.804551:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948794.804558:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009129b91b. 00000400:00000010:2.0:1784948794.804566:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000200:2.0:1784948794.804574:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289653248, portal 25 00000100:00000001:2.0:1784948794.804577:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948794.804579:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451647872 : -115544257903744 : ffff96e9c18d4380) 00000100:00000040:2.0:1784948794.804586:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948794.804597:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948794.804601:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948794.804612:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000400:00000200:1.0:1784948794.804616:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000200:2.0:1784948794.804619:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289653248, offset 0 00000400:00000200:1.0:1784948794.804627:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000400:00000200:0.0:1784948794.804628:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00080000:1.0:1784948794.804636:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1784948794.804637:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948794.804645:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000040:1.0:1784948794.804651:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948794.804679:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1784948794.804685:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000194fc0c0. 00000100:00000040:1.0:1784948794.804686:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000400:00000200:2.0:1784948794.804690:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:0.0:1784948794.804700:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000236111b7. 00000100:00000001:1.0:1784948794.804701:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:1.0:1784948794.804712:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948794.804719:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000200:2.0:1784948794.804719:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436000 00000400:00000010:0.0:1784948794.804720:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000758fd515. 00000020:00000001:1.0:1784948794.804732:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1784948794.804738:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x798d [2] + 0 00000020:00000040:1.0:1784948794.804741:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:2.0:1784948794.804748:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ba88f347. 00000020:00000001:1.0:1784948794.804749:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:2.0:1784948794.804759:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000002ae429a6 00000400:00000200:0.0:1784948794.804768:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000400:00000200:2.0:1784948794.804770:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948794.804780:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948794.804789:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000400:00000200:0.0:1784948794.804798:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948794.804804:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948794.804808:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948794.804812:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000400:00000200:0.0:1784948794.804816:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3260:3260:233:4294967295] 1.2.3.4@tcp LPNI seq info [98:98:8:4294967295] 00000400:00000200:0.0:1784948794.804825:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000400:00000200:0.0:1784948794.804834:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:0.0:1784948794.804844:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000400:00000200:2.0:1784948794.804845:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ae429a6 00000400:00000010:2.0:1784948794.804851:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000002ae429a6. 00000800:00000010:0.0:1784948794.804853:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a3cf4688. 00000100:00000001:2.0:1784948794.804860:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948794.804865:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1784948794.804867:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:2.0:1784948794.804871:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000ace3646f x1871651289653248 msgsize 224 00000800:00000200:0.0:1784948794.804874:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 9 secs later) 00000100:00100000:2.0:1784948794.804888:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948794.804898:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:0.1:1784948794.804898:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784948794.804952:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 9 secs later) 00000100:00000001:2.0:1784948794.804959:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:1.0:1784948794.804960:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000800:00000100:0.0:1784948794.804969:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000400:00000200:2.0:1784948794.804973:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000800:00000010:0.0:1784948794.804974:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a3cf4688. 00000400:00000010:2.0:1784948794.804980:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000400:00000200:0.0:1784948794.804985:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1784948794.804989:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784948794.804996:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000200:2.0:1784948794.805001:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948794.805007:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948794.805020:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000001:2.0:1784948794.805025:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784948794.805030:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000040:2.0:1784948794.805033:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948794.805044:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948794.805055:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1784948794.805064:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:2.0:1784948794.805065:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.805071:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948794.805076:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948794.805078:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000200:0.0:1784948794.805082:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000020:00000040:1.0:1784948794.805085:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:2.0:1784948794.805087:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948794.805097:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000100:00000001:2.0:1784948794.805098:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305734187 00000100:00080000:1.0:1784948794.805100:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000400:00000001:0.0:1784948794.805112:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1784948794.805114:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000010:2.0:1784948794.805114:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000400:00000001:0.0:1784948794.805119:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948794.805127:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000236111b7 00000100:00000001:1.0:1784948794.805128:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.805129:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000010:0.0:1784948794.805137:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000236111b7. 02000000:00000001:1.0:1784948794.805139:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1784948794.805146:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:2.0:1784948794.805147:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305734187 00000100:00000001:3.0:1784948794.805149:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:1.0:1784948794.805150:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948794.805158:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000010:2.0:1784948794.805159:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a3d0cc2f. 02000000:00000001:1.0:1784948794.805161:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948794.805163:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289653248 00000400:00000010:0.0:1784948794.805168:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000194fc0c0. 00000100:00000001:2.0:1784948794.805171:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948794.805172:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1784948794.805174:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948794.805180:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948794.805181:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:2.0:1784948794.805185:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305734187 00000100:00000001:3.0:1784948794.805185:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000010:1.0:1784948794.805193:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e8f14423. 00000400:00000200:0.0:1784948794.805195:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:3.0:1784948794.805195:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948794.805199:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000020:00000040:1.0:1784948794.805202:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784948794.805213:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.805214:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:1.0:1784948794.805216:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1784948794.805217:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 02000000:00000001:1.0:1784948794.805224:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.805225:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305734187 00000100:00100000:3.0:1784948794.805231:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289653248 00000100:00000001:1.0:1784948794.805234:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:2.0:1784948794.805235:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f4af3cce. 00000020:00000001:3.0:1784948794.805242:0:342756:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784948794.805243:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948794.805245:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:1.0:1784948794.805252:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948794.805254:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784948794.805258:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948794.805262:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:1.0:1784948794.805263:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000fed721f2. 00000100:00000001:2.0:1784948794.805264:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948794.805273:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784948794.805275:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.805276:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.805288:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948794.805288:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00100000:1.0:1784948794.805296:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1784948794.805302:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948794.805303:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948794.805309:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948794.805319:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948794.805320:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948794.805325:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948794.805328:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948794.805331:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.805332:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948794.805335:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000002e6ac4ed. 00000100:00000001:2.0:1784948794.805339:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.805344:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e8f14423] to pc [ptlrpcd_rcv+-1] req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948794.805350:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948794.805364:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000002e6ac4ed. 00000020:00000010:3.0:1784948794.805381:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f0e4e38. 00000100:00000001:1.0:1784948794.805385:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.805392:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948794.805397:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948794.805402:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000dc33132b. 00000020:00000040:1.0:1784948794.805406:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948794.805410:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948794.805413:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948794.805416:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948794.805417:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b74af153. 02000000:00000010:1.0:1784948794.805423:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e79c6895. 00000020:00000010:3.0:1784948794.805430:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:1.0:1784948794.805431:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948794.805433:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948794.805442:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948794.805447:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948794.805451:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948794.805451:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948794.805462:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:0.0:1784948794.805467:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948794.805468:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948794.805472:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948794.805476:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.805478:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948794.805495:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948794.805496:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.805497:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc33132b] to pc [ptlrpcd_00_02+2] req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.805501:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948794.805514:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948794.805519:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948794.805525:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948794.805528:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948794.805533:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289653376:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784948794.805541:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784948794.805547:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948794.805548:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948794.805552:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:0.0:1784948794.805555:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948794.805558:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:1.0:1784948794.805560:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11071) 02000000:00000001:0.0:1784948794.805561:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1784948794.805571:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 706 02000000:00000010:0.0:1784948794.805573:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000b9f7c212. 00000100:00000040:3.0:1784948794.805577:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 02000000:00000001:0.0:1784948794.805580:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948794.805583:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165126224 : -115539544425392 : ffff96eada7f3c50) 00000400:00000010:0.0:1784948794.805599:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ecb3ce7f. 00000100:00000001:1.0:1784948794.805599:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948794.805600:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ace3646f x1871651289653248/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:455/0 lens 224/0 e 0 to 0 dl 1784948805 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948794.805607:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:0.0:1784948794.805611:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000236111b7. 00000100:00000001:1.0:1784948794.805615:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948794.805621:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784948794.805626:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289653376, portal 4 00000100:00000001:2.0:1784948794.805630:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.805631:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.805638:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948794.805646:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451655040 : -115544257896576 : ffff96e9c18d5f80) 00000100:00000001:3.0:1784948794.805648:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948794.805677:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000002e6ac4ed. 00000100:00000001:1.0:1784948794.805682:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784948794.805687:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948819 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948794.805687:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948794.805689:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948794.805700:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000010:3.0:1784948794.805702:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000002e6ac4ed. 02000000:00000001:1.0:1784948794.805715:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784948794.805723:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ace3646f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289653248:12345-0@lo:400: 02000000:00000001:1.0:1784948794.805726:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948794.805730:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1784948794.805740:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000dc33132b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289653504:192.168.202.137@tcp:400: 00000100:00000040:0.0:1784948794.805746:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:3.0:1784948794.805747:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289653248 00000100:00000001:1.0:1784948794.805753:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:0.0:1784948794.805758:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f43fd4d9. 00000020:00000001:3.0:1784948794.805760:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:1.0:1784948794.805761:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000200:0.0:1784948794.805766:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289653376, offset 0 02000000:00000001:1.0:1784948794.805768:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948794.805773:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:1.0:1784948794.805775:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948794.805783:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784948794.805784:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948794.805786:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000072ce5ef0. 00000020:00000001:3.0:1784948794.805795:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:0.0:1784948794.805796:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000001:1.0:1784948794.805797:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948794.805805:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000010:1.0:1784948794.805811:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b2f60a65. 00000400:00000200:0.0:1784948794.805815:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784948794.805820:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:1.0:1784948794.805824:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000200:0.0:1784948794.805825:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:3.0:1784948794.805829:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948794.805838:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289653504, portal 10 00000400:00000200:0.0:1784948794.805839:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1784948794.805841:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:0.0:1784948794.805851:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784948794.805852:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1784948794.805853:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948794.805858:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451662208 : -115544257889408 : ffff96e9c18d7b80) 00000400:00000200:0.0:1784948794.805863:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3261:3261:233:4294967295] 192.168.202.137@tcp LPNI seq info [2105:2105:8:4294967295] 00000020:00000001:3.0:1784948794.805865:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.805875:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948794.805878:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948794.805888:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948794.805898:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948794.805948:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784948794.805958:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:1.0:1784948794.805959:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000010:3.0:1784948794.805963:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000077ad1613. 00000400:00000010:1.0:1784948794.805970:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 02000000:00000001:3.0:1784948794.805973:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948794.805979:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289653504, offset 0 00000100:00000001:3.0:1784948794.805983:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948794.805985:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784948794.805995:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948794.805999:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a3cf4688. 00010000:00000040:3.0:1784948794.806004:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289653248 00000400:00000200:1.0:1784948794.806005:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00010000:00000001:3.0:1784948794.806015:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000200:0.0:1784948794.806017:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00010000:00000200:3.0:1784948794.806029:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ace3646f x1871651289653248/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:455/0 lens 224/224 e 0 to 0 dl 1784948805 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948794.806037:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000800:00000200:0.0:1784948794.806052:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:1.0:1784948794.806056:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:0.0:1784948794.806066:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a3cf4688 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1784948794.806071:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00010000:00000001:3.0:1784948794.806074:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948794.806080:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948794.806083:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00001000:3.0:1784948794.806086:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=116 v=5 (1 1 1 1) 00000400:00000200:1.0:1784948794.806093:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784948794.806094:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948794.806096:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948794.806108:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000040:3.0:1784948794.806109:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:1.0:1784948794.806113:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3262:3262:232:4294967295] 192.168.202.137@tcp LPNI seq info [2106:2106:7:4294967295] 00000100:00000001:0.0:1784948794.806122:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305734188 00000100:00000001:3.0:1784948794.806122:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948794.806137:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784948794.806142:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305734187 02000000:00000001:3.0:1784948794.806144:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948794.806148:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948794.806160:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305734188 00000100:00000001:3.0:1784948794.806162:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000200:1.0:1784948794.806170:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948794.806175:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305734187 00000100:00000040:3.0:1784948794.806181:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000010:1.0:1784948794.806186:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000100:00000001:0.0:1784948794.806187:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305734188 00000400:00000010:3.0:1784948794.806192:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000001:0.0:1784948794.806201:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305734187 00000100:00000200:3.0:1784948794.806206:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289653248, offset 224 00000800:00000200:1.0:1784948794.806207:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948794.806212:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305734188 00000800:00000200:1.0:1784948794.806223:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948794.806226:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948794.806227:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305734187 00000800:00000200:1.0:1784948794.806229:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1784948794.806237:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000010:2.0:1784948794.806239:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a3cf4688. 00000100:00000001:0.0:1784948794.806242:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948794.806250:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948794.806252:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948794.806254:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.806260:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948794.806262:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948794.806270:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948794.806272:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:1.0:1784948794.806274:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2048 usecs (3 jiffies) 00000100:00000001:0.0:1784948794.806277:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1784948794.806281:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f43fd4d9 00000100:00000001:0.0:1784948794.806284:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1784948794.806286:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305734149 00000400:00000010:2.0:1784948794.806289:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f43fd4d9. 00000100:00000001:0.0:1784948794.806294:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948794.806299:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305734188 00000100:00000001:0.0:1784948794.806301:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948794.806303:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1784948794.806308:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436000 00000100:00000001:0.0:1784948794.806311:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.806312:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305734149 00000100:00000200:2.0:1784948794.806318:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948819 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.806324:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.806325:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305734188 00000100:00000001:1.0:1784948794.806337:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305734149 00000400:00000200:3.0:1784948794.806338:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x83a5 [1] + 224 00000100:00000001:1.0:1784948794.806348:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305734188 00000100:00000001:1.0:1784948794.806358:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305734149 00000100:00000001:2.0:1784948794.806365:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948794.806366:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305734188 00000400:00000200:3.0:1784948794.806369:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948794.806370:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948794.806373:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948794.806377:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948819 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948794.806383:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948794.806389:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:0.0:1784948794.806395:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.806399:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948794.806405:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948794.806410:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948794.806412:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:0.0:1784948794.806415:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.806427:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948794.806428:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948794.806433:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948794.806435:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948794.806438:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948794.806442:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784948794.806450:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948794.806458:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948794.806468:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948794.806476:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6436080 00000100:00000001:1.0:1784948794.806482:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948794.806484:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000100:0.0:1784948794.806495:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289653376 offset 0 length 520: 4 00000400:00000200:0.0:1784948794.806509:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 0000000023ed5eac is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:0.0:1784948794.806519:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000200:2.0:1784948794.806521:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948794.806532:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000800:00000001:0.0:1784948794.806539:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948794.806543:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948794.806545:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948794.806553:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784948794.806556:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948794.806557:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948794.806566:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948794.806570:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948794.806572:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000800:00000001:1.0:1784948794.806576:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948794.806580:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948794.806582:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000800:00000001:1.0:1784948794.806591:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948794.806594:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:2.0:1784948794.806595:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000400:00000010:2.0:1784948794.806605:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000100:00000001:3.0:1784948794.806605:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948794.806618:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948794.806619:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948794.806623:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948794.806624:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948794.806632:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.806633:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948794.806641:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:0.0:1784948794.806643:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948794.806651:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:3.0:1784948794.806678:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:2.0:1784948794.806681:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948794.806682:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436100 00000400:00000010:0.0:1784948794.806683:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009129b91b. 00000020:00000001:3.0:1784948794.806684:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948794.806697:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948794.806698:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:0.0:1784948794.806699:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:0.0:1784948794.806709:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000100:00000040:3.0:1784948794.806714:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ace3646f x1871651289653248/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:455/0 lens 224/224 e 0 to 0 dl 1784948805 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948794.806719:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784948794.806732:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948794.806733:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948794.806735:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 153808 00000100:00000001:2.0:1784948794.806743:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784948794.806767:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:3.0:1784948794.806778:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ace3646f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289653248:12345-0@lo:400: Request processed in 1122us (1897us total) trans 0 rc 0/0 00000100:00000200:0.0:1784948794.806787:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948794.806794:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948794.806801:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 706 00000400:00000200:1.0:1784948794.806810:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784948794.806813:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948794.806825:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948794.806830:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948794.806831:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:1.0:1784948794.806838:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948794.806840:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948794.806843:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b74af153. 00000100:00000001:0.0:1784948794.806851:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:3.0:1784948794.806857:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 02000000:00000001:0.0:1784948794.806858:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948794.806860:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784948794.806864:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:1.0:1784948794.806869:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000010:3.0:1784948794.806869:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f0e4e38. 00000100:00000001:0.0:1784948794.806870:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948794.806881:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:0.0:1784948794.806883:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948794.806887:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b47dc450. 02000000:00000001:3.0:1784948794.806892:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:0.0:1784948794.806896:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=116 v=5 (5 5 5 5) 00000100:00000040:1.0:1784948794.806897:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000b47dc450 x1871651289653504 msgsize 224 02000000:00000010:3.0:1784948794.806898:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000077ad1613. 02000000:00000001:3.0:1784948794.806952:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:0.0:1784948794.806959:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=116 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948794.806960:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948794.806976:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948794.806977:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00100000:1.0:1784948794.806985:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948794.806987:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948794.806991:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e8e1fecb. 00010000:00000001:0.0:1784948794.806996:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948794.807003:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948794.807003:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000090a0d94e. 00000100:00000001:1.0:1784948794.807006:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948794.807015:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948794.807024:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948794.807027:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948794.807037:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948794.807039:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784948794.807040:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784948794.807047:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289653504 00000100:00000001:0.0:1784948794.807048:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1784948794.807056:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948794.807057:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948794.807062:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:0.0:1784948794.807067:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948794.807069:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948794.807080:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948794.807090:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289653504 00000100:00100000:0.0:1784948794.807093:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289653248:0@lo:400: 00000020:00000001:2.0:1784948794.807096:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948794.807103:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784948794.807104:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948794.807107:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948794.807110:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1784948794.807113:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:0.0:1784948794.807117:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005b93d109 x1871651289653248/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948794.807121:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948794.807130:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948794.807138:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948794.807144:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000010:2.0:1784948794.807145:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001ca1823. 02000000:00000001:0.0:1784948794.807148:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1784948794.807153:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1784948794.807154:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 00000020:00000010:2.0:1784948794.807160:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001ca1823. 02000000:00000001:0.0:1784948794.807163:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948794.807169:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:2.0:1784948794.807170:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000020:00000040:0.0:1784948794.807171:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948794.807181:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000010:2.0:1784948794.807182:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000084b47b3c. 02000000:00000010:0.0:1784948794.807184:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000adf08f9b. 00000020:00000010:2.0:1784948794.807191:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 02000000:00000001:0.0:1784948794.807193:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948794.807198:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948794.807201:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b93d109. 00000100:00000040:2.0:1784948794.807203:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948794.807206:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948794.807209:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948794.807211:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948794.807216:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948794.807218:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948794.807223:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948794.807226:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948794.807231:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.807231:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948794.807238:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948794.807240:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948794.807243:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948794.807245:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.807267:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948794.807281:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948794.807283:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948794.807289:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 682 00000100:00000040:2.0:1784948794.807293:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948794.807296:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451656832 : -115544257894784 : ffff96e9c18d6680) 00000100:00000040:2.0:1784948794.807303:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b47dc450 x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:455/0 lens 224/0 e 0 to 0 dl 1784948805 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948794.807315:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948794.807318:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001ca1823. 00000100:00000001:2.0:1784948794.807321:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948794.807323:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001ca1823. 00000100:00100000:2.0:1784948794.807329:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b47dc450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289653504:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784948794.807334:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289653504 00000020:00000001:2.0:1784948794.807338:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948794.807342:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948794.807344:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948794.807347:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948794.807349:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784948794.807353:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948794.807356:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948794.807358:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948794.807360:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948794.807362:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948794.807365:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948794.807368:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948794.807371:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948794.807375:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002269aa10. 02000000:00000001:2.0:1784948794.807378:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.807380:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948794.807383:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948794.807385:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289653504 00010000:00000001:2.0:1784948794.807388:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948794.807392:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b47dc450 x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:455/0 lens 224/224 e 0 to 0 dl 1784948805 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948794.807401:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948794.807403:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948794.807406:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=111 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948794.807411:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948794.807414:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948794.807416:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948794.807419:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948794.807421:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948794.807424:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948794.807427:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948794.807430:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000200:2.0:1784948794.807433:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289653504, offset 224 00000400:00000200:2.0:1784948794.807438:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948794.807447:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948794.807453:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3263:3263:233:4294967295] 192.168.202.137@tcp LPNI seq info [2107:2107:8:4294967295] 00000400:00000200:2.0:1784948794.807464:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948794.807470:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948794.807475:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000069a945c8. 00000800:00000200:2.0:1784948794.807480:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948794.807487:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784948794.807491:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000069a945c8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948794.807506:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948794.807510:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948794.807513:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948794.807515:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948794.807518:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948794.807521:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b47dc450 x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:455/0 lens 224/224 e 0 to 0 dl 1784948805 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948794.807533:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b47dc450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289653504:12345-192.168.202.137@tcp:400: Request processed in 216us (635us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948794.807542:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 682 00000100:00000040:2.0:1784948794.807545:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948794.807549:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948794.807552:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948794.807557:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000084b47b3c. 00000020:00000010:2.0:1784948794.807561:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948794.807572:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000020:00000040:2.0:1784948794.807577:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948794.807581:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948794.807626:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948794.807632:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000069a945c8. 00000800:00000001:2.0:1784948794.807636:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948794.807640:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948794.807649:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948794.807650:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948794.807684:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948794.807689:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000800:00000001:2.0:1784948794.807692:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948794.807695:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000400:00000200:2.0:1784948794.807707:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948794.807708:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948794.807714:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948794.807726:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436100 00000400:00000200:2.0:1784948794.807734:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x83b9 [1] + 224 00000800:00000001:2.0:1784948794.807742:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948794.807749:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948794.807751:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948794.807756:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948794.807761:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948794.807765:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948794.807776:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948794.807806:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948794.807813:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948794.807817:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.807864:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948794.807871:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948794.807878:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948794.807881:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948794.807884:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b2f60a65. 00000400:00000200:1.0:1784948794.807893:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:1.0:1784948794.807897:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000100:00000001:1.0:1784948794.807901:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948794.807985:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948794.808004:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948794.808015:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948794.808017:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.808020:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948794.808022:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948794.808025:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948794.808027:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948794.808030:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948794.808036:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=111 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948794.808040:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=111 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948794.808043:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948794.808045:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948794.808047:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948794.808052:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948794.808061:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.808065:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948794.808067:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948794.808069:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948794.808071:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948794.808074:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948794.808082:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948794.808084:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948794.808086:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948794.808089:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948794.808106:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000dc33132b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289653504:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948794.808112:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.808115:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948794.808117:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000dc33132b x1871651289653504/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948794.808124:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948794.808126:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948794.808129:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000072ce5ef0. 02000000:00000001:1.0:1784948794.808133:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948794.808135:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948794.808137:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948794.808139:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948794.808141:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e79c6895. 02000000:00000001:1.0:1784948794.808145:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948794.808146:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948794.808148:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000dc33132b. 00000100:00000001:1.0:1784948794.808151:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948794.808153:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.808155:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948794.808159:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948794.808162:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948794.808164:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948794.808166:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948794.848998:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305734188, at 4305734189 00000100:00000010:0.0:1784948794.849014:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000100:00000040:1.0:1784948794.849028:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305734188, at 4305734189 00000100:00000040:3.0:1784948794.849028:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305734188, at 4305734189 00000100:00000040:2.0:1784948794.849036:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305734188, at 4305734189 00000100:00000010:1.0:1784948794.849046:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a3d0cc2f. 00000100:00000010:3.0:1784948794.849051:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f4af3cce. 00000100:00000010:2.0:1784948794.849054:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000800:00000100:2.0:1784948794.996075:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.6@tcp 00000800:00000010:2.0:1784948794.996081:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000075354f3d. 00000400:00000200:2.0:1784948794.996089:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948794.996093:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948794.996099:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11068 health :900 00000400:00000200:2.0:1784948794.996104:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948794.996107:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:2.0:1784948794.996111:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:0.0:1784948794.996189:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948794.996199:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2861 00000400:00000001:0.0:1784948794.996208:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948794.996210:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948794.996213:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:0.0:1784948794.996217:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000400:00000200:0.0:1784948794.996233:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948794.996237:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:0.0:1784948794.996243:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006841dfc4. 00000400:00000200:0.0:1784948794.996249:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-110 00000400:00000200:0.0:1784948794.996252:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -110 00000400:00000200:0.0:1784948794.996257:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00010000:00000001:2.0:1784948795.635970:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948795.635976:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948795.635982:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948795.635983:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948795.636057:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948795.636061:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948795.636063:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:1.0:1784948796.020053:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:2.0:1784948796.020057:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.4@tcp 00000400:00000010:1.0:1784948796.020064:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000057de27a4. 00000800:00000010:2.0:1784948796.020066:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ea573232. 00000400:00000200:2.0:1784948796.020077:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:1.0:1784948796.020078:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009f047fe3. 00000400:00000010:1.0:1784948796.020088:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000200:2.0:1784948796.020093:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784948796.020099:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000003b6cd72. 00000400:00000200:1.0:1784948796.020108:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948796.020108:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11069 health :800 00000400:00000200:2.0:1784948796.020121:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948796.020129:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:1.0:1784948796.020130:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948796.020139:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948796.020142:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3264:3264:234:4294967295] 192.168.202.137@tcp LPNI seq info [2108:2108:8:4294967295] 00000400:00000200:1.0:1784948796.020149:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948796.020155:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948796.020160:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948796.020164:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000800:00000010:1.0:1784948796.020167:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000800:00000200:1.0:1784948796.020177:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948796.020187:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948796.020190:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 2, nob 96 niov 1 nkiov 0 00000400:00000010:1.0:1784948796.020214:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 00000000de58da65. 00000800:00000100:2.0:1784948796.020216:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.10@tcp 00000400:02000040:1.0:1784948796.020222:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000800:00000010:2.0:1784948796.020225:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000040401747. 00000400:00000200:2.0:1784948796.020232:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948796.020239:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948796.020242:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948796.020248:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2861 00000400:00000200:2.0:1784948796.020248:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948796.020252:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000001:0.0:1784948796.020254:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1784948796.020256:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000001:0.0:1784948796.020257:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948796.020260:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000400:00000010:0.0:1784948796.020264:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000400:00000200:0.0:1784948796.020268:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948796.020274:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:0.0:1784948796.020278:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ee879bbd. 00000400:00000200:0.0:1784948796.020285:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-110 00000400:00000200:0.0:1784948796.020289:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -110 00000400:00000200:0.0:1784948796.020294:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:0.0:1784948796.020299:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2861 00000400:00000001:0.0:1784948796.020302:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948796.020304:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948796.020306:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000400:00000010:0.0:1784948796.020308:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000400:00000200:0.0:1784948796.020312:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948796.020315:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:0.0:1784948796.020318:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ec8543b3. 00000400:00000200:0.0:1784948796.020326:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-110 00000400:00000200:0.0:1784948796.020329:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -110 00000400:00000200:0.0:1784948796.020334:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00000800:00000001:0.0:1784948796.020381:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948796.020394:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948796.020397:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784948796.020401:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948796.020407:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000800:00000001:0.0:1784948796.020410:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948796.020413:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948796.020424:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948796.020425:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948796.020430:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784948796.020435:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:0.0:1784948796.020441:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948796.020450:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948796.020460:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948796.020465:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3265:3265:236:4294967295] 192.168.202.137@tcp LPNI seq info [2109:2109:8:4294967295] 00000400:00000200:0.0:1784948796.020473:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948796.020477:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948796.020481:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000595fa8f8. 00000800:00000200:0.0:1784948796.020485:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948796.020489:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948796.020492:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000595fa8f8 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784948796.020546:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948796.020564:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948796.020567:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000595fa8f8. 00000400:00000200:0.0:1784948796.020572:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948796.020575:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948796.020578:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948796.020579:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948796.020585:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948796.020587:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948796.020591:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948796.020598:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948796.020613:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x83cd 00000800:00000001:2.0:1784948796.020619:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948796.020625:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948796.020627:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948796.020631:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948796.020637:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:2.0:1784948796.020639:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000400:00000200:2.0:1784948796.020643:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948796.020665:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000057de27a4. 00000400:00000010:2.0:1784948796.020670:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009f047fe3. 00000800:00000001:2.0:1784948796.020676:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948796.020679:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948796.023235:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 00000000de58da65. 00010000:00000001:2.0:1784948796.660040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948796.660049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948796.660056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948796.660058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948796.660145:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948796.660151:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948796.660156:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948797.684009:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948797.684018:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948797.684025:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948797.684027:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948797.684103:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948797.684110:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948797.684115:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1784948798.068074:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:2.0:1784948798.068085:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000800:00000200:2.0:1784948798.068092:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1784948798.068324:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784948798.708033:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948798.708041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948798.708050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948798.708054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948798.708135:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948798.708142:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948798.708146:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948799.732048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948799.732056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948799.732064:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948799.732068:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948799.732071:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948799.732075:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948799.732079:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948799.732082:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948799.732084:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948799.732087:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948799.732176:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948799.732184:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948799.732190:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948799.924038:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948799.924052:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948799.924054:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948799.924060:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948799.924064:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948799.924068:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.924071:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948799.924097:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948799.924108:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948799.924117:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000020:00000040:0.0:1784948799.924125:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948799.924131:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948799.924135:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948799.924141:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948799.924150:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000595fa8f8. 00000100:00000001:0.0:1784948799.924158:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948799.924169:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948799.924184:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948799.924187:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948799.924193:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c39699a8] to pc [ptlrpcd_00_03+3] req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948799.924231:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948799.924237:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948799.924243:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948799.924248:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948799.924254:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948799.924260:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009778919c. 00000020:00000040:0.0:1784948799.924265:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948799.924271:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948799.924275:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948799.924279:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948799.924286:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d1b12746. 00000100:00000001:1.0:1784948799.924288:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948799.924294:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948799.924296:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:2.0:1784948799.924303:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948799.924306:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948799.924312:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948799.924314:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948799.924315:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948799.924321:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948799.924328:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948799.924334:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948799.924340:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948799.924349:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1784948799.924351:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948799.924359:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.924376:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784948799.924379:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009778919c] to pc [ptlrpcd_00_00+0] req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948799.924381:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948799.924390:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948799.924396:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948799.924404:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948799.924412:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289653632:0@lo:400: 00000100:00000001:1.0:1784948799.924419:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948799.924426:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948799.924429:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948799.924432:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948799.924436:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948799.924439:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dfc732c2. 02000000:00000001:1.0:1784948799.924445:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948799.924450:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11076) 00000400:00000010:1.0:1784948799.924453:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b2f60a65. 00000400:00000010:1.0:1784948799.924469:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000200:1.0:1784948799.924477:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289653632, portal 25 00000100:00000001:1.0:1784948799.924482:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948799.924485:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558302976 : -115544151248640 : ffff96e9c7e8b100) 00000100:00000001:2.0:1784948799.924488:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948799.924495:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948799.924497:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:3.0:1784948799.924502:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948799.924512:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948799.924513:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.924517:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948799.924526:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948799.924529:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784948799.924533:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289653632, offset 0 00000100:00000001:2.0:1784948799.924533:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948799.924542:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948799.924550:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948799.924551:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948799.924556:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948799.924559:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948799.924569:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948799.924570:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784948799.924588:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948799.924595:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948799.924596:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436180 02000000:00000001:2.0:1784948799.924603:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948799.924608:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948799.924644:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x79a9 [2] + 0 02000000:00000001:2.0:1784948799.924654:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948799.924658:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000068d5a5a2. 00000100:00100000:2.0:1784948799.924666:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000009778919c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289653760:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784948799.924670:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000216d6fb0 00000100:00000001:2.0:1784948799.924673:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948799.924680:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948799.924682:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948799.924686:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784948799.924687:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784948799.924695:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 00000400:00000200:1.0:1784948799.924704:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:2.0:1784948799.924705:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948799.924718:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba88f347. 00000400:00000200:1.0:1784948799.924721:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000216d6fb0 00000400:00000010:1.0:1784948799.924730:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000216d6fb0. 00000100:00000001:1.0:1784948799.924740:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000400:00000010:2.0:1784948799.924744:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000200:1.0:1784948799.924745:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:2.0:1784948799.924756:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289653760, portal 10 00000100:00000040:1.0:1784948799.924758:0:337121:0:(events.c:337:request_in_callback()) incoming req@000000006f58d1a4 x1871651289653632 msgsize 224 00000100:00000001:2.0:1784948799.924764:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948799.924769:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558294016 : -115544151257600 : ffff96e9c7e88e00) 00000100:00100000:1.0:1784948799.924780:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948799.924787:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948799.924790:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784948799.924804:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948799.924809:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948799.924813:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000100:00000200:2.0:1784948799.924815:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289653760, offset 0 00000400:00000200:2.0:1784948799.924821:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948799.924824:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784948799.924837:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948799.924846:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948799.924852:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948799.924859:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948799.924864:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784948799.924870:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3266:3266:236:4294967295] 192.168.202.137@tcp LPNI seq info [2110:2110:8:4294967295] 00000100:00000001:3.0:1784948799.924873:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948799.924879:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289653632 02000000:00000001:3.0:1784948799.924887:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948799.924893:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784948799.924894:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948799.924899:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948799.924947:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948799.924954:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289653632 00000400:00000200:1.0:1784948799.924955:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000800:00000200:2.0:1784948799.924960:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948799.924962:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000020:00000001:3.0:1784948799.924962:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948799.924970:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784948799.924975:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1784948799.924977:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000010:2.0:1784948799.924980:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d5a127c9. 00000020:00000040:3.0:1784948799.924983:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000200:1.0:1784948799.924987:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948799.924994:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784948799.925006:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:2.0:1784948799.925009:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948799.925018:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948799.925030:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1784948799.925031:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000a3980217. 00000800:00000200:2.0:1784948799.925040:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:1.0:1784948799.925041:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948799.925042:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1784948799.925053:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d5a127c9 type 1, nob 320 niov 1 nkiov 1 00000020:00000010:3.0:1784948799.925057:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000a3980217. 00000020:00000010:3.0:1784948799.925075:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eb3e5fa2. 00000100:00000001:1.0:1784948799.925076:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.925085:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784948799.925086:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b74af153. 00000100:00000001:2.0:1784948799.925091:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948799.925096:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:1.0:1784948799.925099:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948799.925105:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2051 usecs (3 jiffies) 00000100:00000001:1.0:1784948799.925111:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305739307 00000100:00000040:3.0:1784948799.925111:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948799.925117:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305739268 00000100:00000001:3.0:1784948799.925121:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948799.925129:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000010:1.0:1784948799.925130:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:3.0:1784948799.925137:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.925147:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948799.925151:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.925164:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305739307 00000100:00000001:2.0:1784948799.925165:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305739307 00000100:00000001:3.0:1784948799.925173:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948799.925180:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305739268 00000100:00000010:1.0:1784948799.925181:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000001:3.0:1784948799.925197:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948799.925198:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948799.925205:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948799.925215:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305739307 00000100:00000001:2.0:1784948799.925216:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305739307 00000100:00100000:3.0:1784948799.925217:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 707 00000100:00000040:3.0:1784948799.925231:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948799.925233:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305739268 00000100:00000010:1.0:1784948799.925235:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000c54be64. 00000100:00000001:3.0:1784948799.925238:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165114960 : -115539544436656 : ffff96eada7f1050) 00000100:00000001:1.0:1784948799.925252:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000040:3.0:1784948799.925254:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006f58d1a4 x1871651289653632/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:460/0 lens 224/0 e 0 to 0 dl 1784948810 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948799.925266:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305739307 00000100:00000001:2.0:1784948799.925267:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305739307 00000100:00000001:3.0:1784948799.925275:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:1.0:1784948799.925281:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000092889e7d. 00000100:00000001:2.0:1784948799.925283:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305739268 00000020:00000010:3.0:1784948799.925285:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000a3980217. 00000800:00000200:0.0:1784948799.925293:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948799.925295:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948799.925295:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000010:0.0:1784948799.925303:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d5a127c9. 00000020:00000010:3.0:1784948799.925307:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000a3980217. 00000100:00000001:1.0:1784948799.925309:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948799.925310:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305739307 00000400:00000200:0.0:1784948799.925316:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948799.925319:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948799.925322:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784948799.925325:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006f58d1a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289653632:12345-0@lo:400: 00000100:00000001:1.0:1784948799.925329:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948799.925335:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948799.925336:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.925341:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948799.925345:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289653632 00000100:00000001:2.0:1784948799.925349:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948799.925354:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784948799.925357:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000100:00000001:2.0:1784948799.925358:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948799.925361:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:0.0:1784948799.925364:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000020:00000001:3.0:1784948799.925367:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948799.925375:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784948799.925379:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1784948799.925380:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784948799.925389:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948799.925392:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948799.925395:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784948799.925398:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948799.925399:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948799.925405:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948799.925412:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948799.925416:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948799.925429:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948799.925440:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784948799.925442:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948799.925442:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:0.0:1784948799.925453:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948799.925457:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948799.925463:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948799.925473:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 00000400:00000200:1.0:1784948799.925474:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:3.0:1784948799.925483:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948799.925494:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948799.925498:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948799.925499:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948799.925510:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000040:3.0:1784948799.925511:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289653632 00000400:00000200:1.0:1784948799.925512:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436200 00010000:00000001:3.0:1784948799.925520:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948799.925532:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006f58d1a4 x1871651289653632/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:460/0 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948799.925543:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 154032 00010000:00000001:3.0:1784948799.925565:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000800:00000001:1.0:1784948799.925569:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:3.0:1784948799.925570:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948799.925580:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=121 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948799.925586:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:1.0:1784948799.925589:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948799.925592:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:1.0:1784948799.925597:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948799.925599:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948799.925606:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:1.0:1784948799.925635:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784948799.925638:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.925646:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948799.925646:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:1.0:1784948799.925654:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:3.0:1784948799.925662:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000010:1.0:1784948799.925667:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000dc33132b. 00000400:00000010:3.0:1784948799.925670:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000040:1.0:1784948799.925675:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000dc33132b x1871651289653760 msgsize 224 00000100:00000200:3.0:1784948799.925680:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289653632, offset 224 00000100:00100000:1.0:1784948799.925692:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:3.0:1784948799.925699:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948799.925712:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784948799.925717:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948799.925725:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436180 00000800:00000001:1.0:1784948799.925732:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948799.925734:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x83d1 [1] + 224 00000800:00000001:1.0:1784948799.925740:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948799.925746:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948799.925756:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948799.925763:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948799.925770:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948799.925777:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948799.925784:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289653760 00000100:00000040:3.0:1784948799.925790:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948799.925791:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948799.925797:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948799.925802:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948799.925812:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948799.925818:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289653760 00000020:00000001:0.0:1784948799.925822:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948799.925824:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948799.925826:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948799.925830:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:3.0:1784948799.925831:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1784948799.925835:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948799.925841:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784948799.925844:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000100:00000001:0.0:1784948799.925846:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:3.0:1784948799.925849:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000020:00000010:0.0:1784948799.925857:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:3.0:1784948799.925857:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948799.925863:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948799.925866:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948799.925872:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948799.925873:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000100:00000001:1.0:1784948799.925883:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948799.925885:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000010:0.0:1784948799.925886:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000100:00000001:1.0:1784948799.925890:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1784948799.925895:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948799.925902:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948799.925929:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948799.925934:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948799.925938:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b2f60a65. 00000400:00000200:1.0:1784948799.925944:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000020:00000001:3.0:1784948799.925947:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000010:0.0:1784948799.925948:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000080a10e75. 00000400:00000010:1.0:1784948799.925951:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000040:3.0:1784948799.925959:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006f58d1a4 x1871651289653632/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:460/0 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948799.925962:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784948799.925967:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000200:1.0:1784948799.925974:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948799.925989:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948799.926001:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948799.926010:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:3.0:1784948799.926012:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006f58d1a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289653632:12345-0@lo:400: Request processed in 730us (1246us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948799.926019:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948799.926037:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948799.926038:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00100000:3.0:1784948799.926044:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 707 00000100:00000001:0.0:1784948799.926046:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948799.926052:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948799.926055:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948799.926066:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948799.926071:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948799.926079:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b74af153. 00000100:00000001:0.0:1784948799.926082:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948799.926091:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000020:00000010:3.0:1784948799.926104:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eb3e5fa2. 00000100:00000001:1.0:1784948799.926112:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:3.0:1784948799.926116:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948799.926118:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948799.926120:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948799.926125:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784948799.926127:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948799.926131:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000010:3.0:1784948799.926132:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 02000000:00000001:1.0:1784948799.926142:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00100000:0.0:1784948799.926143:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 683 02000000:00000001:3.0:1784948799.926143:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784948799.926148:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948799.926151:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948799.926152:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948799.926154:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948799.926154:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948799.926157:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948799.926165:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948799.926166:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948799.926167:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451662208 : -115544257889408 : ffff96e9c18d7b80) 02000000:00000001:1.0:1784948799.926168:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948799.926179:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948799.926179:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ea556a29. 00000100:00001000:1.0:1784948799.926184:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=121 v=5 (5 5 5 5) 00000100:00000040:0.0:1784948799.926191:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000dc33132b x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:460/0 lens 224/0 e 0 to 0 dl 1784948810 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948799.926191:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007c7d797b. 00000100:00001000:1.0:1784948799.926199:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=121 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948799.926215:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948799.926222:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948799.926233:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784948799.926237:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:1.0:1784948799.926240:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948799.926248:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:1.0:1784948799.926249:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948799.926261:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948799.926268:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948799.926270:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000100:00100000:0.0:1784948799.926289:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000dc33132b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289653760:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948799.926293:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784948799.926302:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289653760 00000100:00000001:1.0:1784948799.926303:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1784948799.926313:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784948799.926313:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948799.926324:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:1.0:1784948799.926327:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948799.926330:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948799.926340:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948799.926346:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00100000:1.0:1784948799.926350:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289653632:0@lo:400: 00000020:00000001:0.0:1784948799.926355:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948799.926364:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948799.926365:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948799.926374:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948799.926375:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784948799.926381:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:1.0:1784948799.926383:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c39699a8 x1871651289653632/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948799.926388:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948799.926396:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948799.926404:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948799.926404:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948799.926412:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1784948799.926413:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948799.926418:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dfc732c2. 02000000:00000010:0.0:1784948799.926421:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000051537263. 02000000:00000001:1.0:1784948799.926427:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1784948799.926428:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948799.926435:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:0.0:1784948799.926436:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948799.926441:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948799.926443:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948799.926452:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000040:0.0:1784948799.926453:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289653760 02000000:00000010:1.0:1784948799.926460:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000595fa8f8. 00010000:00000001:0.0:1784948799.926461:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:1.0:1784948799.926471:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00010000:00000200:0.0:1784948799.926475:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000dc33132b x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:460/0 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948799.926477:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948799.926482:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c39699a8. 00000100:00000001:1.0:1784948799.926490:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948799.926495:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948799.926502:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948799.926508:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784948799.926513:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1784948799.926514:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948799.926519:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948799.926522:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=116 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948799.926527:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948799.926531:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784948799.926533:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948799.926540:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948799.926545:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948799.926548:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948799.926550:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948799.926554:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948799.926558:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948799.926561:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 00000100:00000200:0.0:1784948799.926565:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289653760, offset 224 00000400:00000200:0.0:1784948799.926571:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948799.926581:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948799.926587:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3267:3267:236:4294967295] 192.168.202.137@tcp LPNI seq info [2111:2111:8:4294967295] 00000400:00000200:0.0:1784948799.926598:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948799.926604:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948799.926608:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b84187a1. 00000800:00000200:0.0:1784948799.926640:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948799.926646:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948799.926650:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b84187a1 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948799.926670:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948799.926676:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948799.926680:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948799.926682:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948799.926686:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948799.926693:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000dc33132b x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:460/0 lens 224/224 e 0 to 0 dl 1784948810 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948799.926710:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000dc33132b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289653760:12345-192.168.202.137@tcp:400: Request processed in 472us (1032us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948799.926725:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 683 00000100:00000040:0.0:1784948799.926730:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948799.926734:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948799.926737:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948799.926742:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000080a10e75. 00000020:00000010:0.0:1784948799.926749:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948799.926755:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 00000020:00000040:0.0:1784948799.926761:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948799.926765:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948799.926852:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948799.926854:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784948799.926858:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b84187a1. 00000800:00000001:0.0:1784948799.926864:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948799.926864:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948799.926869:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948799.926873:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948799.926873:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948799.926880:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000400:00000200:0.0:1784948799.926882:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:2.0:1784948799.926883:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000100:00000001:2.0:1784948799.926889:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948799.926892:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784948799.926896:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436200 00000400:00000200:0.0:1784948799.926937:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x83d9 [1] + 224 00000800:00000001:0.0:1784948799.926948:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948799.926961:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948799.926965:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948799.926973:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948799.926981:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948799.926987:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948799.927000:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948799.927023:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948799.927033:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948799.927039:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948799.927079:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948799.927084:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948799.927090:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948799.927093:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948799.927096:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ba88f347. 00000400:00000200:2.0:1784948799.927103:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:2.0:1784948799.927106:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:2.0:1784948799.927111:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948799.927118:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948799.927131:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948799.927139:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948799.927141:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948799.927144:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948799.927147:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948799.927149:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948799.927151:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948799.927153:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948799.927159:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=116 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948799.927163:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=116 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948799.927166:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948799.927168:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948799.927170:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948799.927174:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948799.927182:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948799.927186:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948799.927188:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948799.927190:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948799.927192:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948799.927195:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948799.927203:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948799.927205:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948799.927207:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948799.927210:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948799.927220:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000009778919c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289653760:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948799.927224:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948799.927227:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948799.927229:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000009778919c x1871651289653760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948815 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948799.927237:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948799.927238:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948799.927240:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:2.0:1784948799.927244:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948799.927246:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948799.927248:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948799.927251:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948799.927253:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d1b12746. 02000000:00000001:2.0:1784948799.927256:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948799.927257:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948799.927260:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000009778919c. 00000100:00000001:2.0:1784948799.927263:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948799.927265:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948799.927267:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948799.927272:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948799.927274:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948799.927276:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948799.927278:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948799.968024:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305739307, at 4305739308 00000100:00000010:1.0:1784948799.968037:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000040:3.0:1784948799.968043:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305739307, at 4305739308 00000100:00000040:0.0:1784948799.968056:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305739307, at 4305739308 00000100:00000040:2.0:1784948799.968060:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305739307, at 4305739308 00000100:00000010:3.0:1784948799.968060:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000092889e7d. 00000100:00000010:2.0:1784948799.968078:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000c54be64. 00000100:00000010:0.0:1784948799.968079:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00010000:00000001:2.0:1784948800.755970:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948800.755977:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948800.755983:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948800.755984:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948800.756030:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948800.756036:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948800.756039:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948800.947946:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948800.947956:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948800.947958:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948800.947961:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948800.947963:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948800.947965:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948800.947968:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948801.780028:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948801.780037:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948801.780044:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948801.780046:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948801.780136:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948801.780144:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948801.780150:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948801.971991:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948801.972006:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948801.972008:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948801.972015:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948801.972019:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948801.972023:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948801.972026:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948802.804042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948802.804054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948802.804064:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948802.804066:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948802.804145:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948802.804152:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948802.804157:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948802.996003:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948802.996039:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948802.996042:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948802.996049:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948802.996053:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948802.996058:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948802.996061:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948803.828057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948803.828067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948803.828075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948803.828077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948803.828157:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948803.828164:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948803.828170:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948804.020022:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948804.020039:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948804.020041:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948804.020048:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948804.020052:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948804.020061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948804.020064:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1784948804.532068:0:344025:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.14:988 00000400:00000100:2.0:1784948804.532086:0:343873:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.8:988 00000400:00000100:1.0:1784948804.532095:0:343579:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.10:988 00000400:00000100:3.0:1784948804.532109:0:344024:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.12:988 00000400:00000100:0.0:1784948804.532159:0:344025:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.14@tcp at host 111.222.173.14:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784948804.532168:0:343873:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.8@tcp at host 111.222.173.8:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784948804.532175:0:343579:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.10@tcp at host 111.222.173.10:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784948804.532178:0:344025:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.14@tcp: down 00000400:00000100:3.0:1784948804.532183:0:344024:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.12@tcp at host 111.222.173.12:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948804.532187:0:343873:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.8@tcp: down 00000400:00000200:0.0:1784948804.532193:0:344025:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784948804.532195:0:343579:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.10@tcp: down 00000400:00000200:3.0:1784948804.532204:0:344024:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.12@tcp: down 00000400:00000200:2.0:1784948804.532209:0:343873:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1784948804.532212:0:344025:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000400:00000200:1.0:1784948804.532217:0:343579:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1784948804.532222:0:343873:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000800:00000010:0.0:1784948804.532224:0:344025:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000800:00000010:2.0:1784948804.532228:0:343873:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000820f02b3. 00000400:00000200:3.0:1784948804.532228:0:344024:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948804.532241:0:343873:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:3.0:1784948804.532241:0:344024:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000400:00000200:0.0:1784948804.532242:0:344025:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000010:3.0:1784948804.532246:0:344024:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000662152ce. 00000400:00000200:3.0:1784948804.532258:0:344024:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948804.532260:0:343873:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948804.532264:0:344025:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948804.532275:0:344024:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948804.532276:0:343873:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11077 health :900 00000400:00000200:2.0:1784948804.532289:0:343873:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948804.532294:0:344025:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948804.532294:0:344024:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948804.532297:0:343873:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:3.0:1784948804.532310:0:344024:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:0.0:1784948804.532313:0:344025:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:2.0:1784948804.532314:0:343873:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:3.0:1784948804.532341:0:344024:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:0.0:1784948804.532355:0:344025:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:1.0:1784948804.532405:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784948804.532416:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000001:1.0:1784948804.532427:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948804.532431:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948804.532435:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784948804.532440:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000400:00000200:1.0:1784948804.532448:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948804.532457:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:1.0:1784948804.532464:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004041a48e. 00000400:00000200:1.0:1784948804.532477:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000100:3.0:1784948804.532477:0:343908:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.6:988 00000400:00000200:1.0:1784948804.532490:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:1.0:1784948804.532500:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000400:00000200:1.0:1784948804.532529:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000100:2.0:1784948804.532531:0:343754:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.2:988 00000400:00000001:1.0:1784948804.532539:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948804.532545:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948804.532549:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000100:3.0:1784948804.532551:0:343908:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.6@tcp at host 111.222.173.6:988 took too long: that node may be hung or experiencing high load. 00000400:00000010:1.0:1784948804.532557:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000400:00000200:1.0:1784948804.532566:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948804.532568:0:343908:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.6@tcp: down 00000400:00000100:0.0:1784948804.532611:0:343916:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.4:988 00000400:00000200:1.0:1784948804.532613:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:1.0:1784948804.532639:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eb983bef. 00000400:00000200:1.0:1784948804.532648:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:3.0:1784948804.532650:0:343908:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:2.0:1784948804.532655:0:343754:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.2@tcp at host 111.222.173.2:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948804.532656:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:1.0:1784948804.532672:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000400:00000200:2.0:1784948804.532676:0:343754:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.2@tcp: down 00000400:00000200:1.0:1784948804.532683:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000100:0.0:1784948804.532684:0:343916:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.4@tcp at host 111.222.173.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948804.532686:0:343754:0:(router.c:1888:lnet_notify()) Old news 00000400:00000001:1.0:1784948804.532691:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948804.532697:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948804.532701:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000800:00000100:2.0:1784948804.532702:0:343754:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000400:00000200:0.0:1784948804.532703:0:343916:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.4@tcp: down 00000400:00000010:1.0:1784948804.532706:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000400:00000200:0.0:1784948804.532714:0:343916:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:2.0:1784948804.532714:0:343754:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000635133da. 00000400:00000200:1.0:1784948804.532715:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948804.532724:0:343754:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:0.0:1784948804.532725:0:343916:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000400:00000200:1.0:1784948804.532725:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000800:00000010:0.0:1784948804.532735:0:343916:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000353a5c12. 00000400:00000010:1.0:1784948804.532736:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000047e19e91. 00000400:00000200:2.0:1784948804.532741:0:343754:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948804.532746:0:343916:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948804.532749:0:343754:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948804.532751:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:2.0:1784948804.532759:0:343754:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:0.0:1784948804.532761:0:343916:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948804.532761:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:2.0:1784948804.532768:0:343754:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:0.0:1784948804.532781:0:343916:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948804.532788:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00000400:00000200:0.0:1784948804.532789:0:343916:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:0.0:1784948804.532798:0:343916:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:1.0:1784948804.532808:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:1.0:1784948804.532813:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948804.532815:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948804.532818:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784948804.532822:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000400:00000200:1.0:1784948804.532827:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948804.532834:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:1.0:1784948804.532839:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000013f68999. 00000400:00000200:1.0:1784948804.532847:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:1.0:1784948804.532852:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:1.0:1784948804.532858:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000400:00000200:1.0:1784948804.532864:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:1.0:1784948804.532868:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948804.532871:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948804.532873:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:1.0:1784948804.532876:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000400:00000200:1.0:1784948804.532881:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948804.532885:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:1.0:1784948804.532889:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000461737cb. 00000400:00000200:1.0:1784948804.532894:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:1.0:1784948804.532898:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:1.0:1784948804.532902:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00010000:00000001:2.0:1784948804.852000:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948804.852009:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948804.852016:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948804.852018:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948804.852174:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948804.852181:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948804.852185:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948805.044031:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948805.044046:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948805.044048:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948805.044055:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948805.044058:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948805.044062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.044066:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948805.044098:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948805.044107:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948805.044116:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000098475f18. 00000020:00000040:2.0:1784948805.044124:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948805.044130:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948805.044143:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948805.044150:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948805.044160:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c03b4f3a. 00000100:00000001:2.0:1784948805.044168:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948805.044190:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948805.044206:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948805.044209:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948805.044215:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098475f18] to pc [ptlrpcd_00_01+1] req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948805.044257:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948805.044260:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948805.044264:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948805.044268:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948805.044273:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948805.044277:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b8226094. 00000020:00000040:2.0:1784948805.044280:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948805.044283:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948805.044285:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948805.044287:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.044293:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1784948805.044293:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000762a0fe3. 00000100:00000001:0.0:1784948805.044301:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948805.044302:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.044308:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948805.044314:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784948805.044323:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948805.044340:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948805.044342:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948805.044350:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8226094] to pc [ptlrpcd_00_02+2] req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948805.044377:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948805.044385:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948805.044392:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.044399:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1784948805.044401:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11082) 00000100:00000001:0.0:1784948805.044408:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.044418:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948805.044420:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1784948805.044424:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 02000000:00000001:0.0:1784948805.044429:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948805.044430:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948805.044443:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.044443:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948805.044449:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948805.044455:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948805.044457:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948805.044458:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000098475f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289653888:0@lo:400: 00000100:00000001:3.0:1784948805.044458:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948805.044464:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948805.044468:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.044473:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1784948805.044478:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948805.044485:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948805.044494:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.044497:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948805.044503:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948805.044507:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1784948805.044509:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948805.044515:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948805.044518:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948805.044521:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 00000100:00100000:3.0:1784948805.044524:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289654016:192.168.202.137@tcp:400: 02000000:00000001:0.0:1784948805.044533:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.044534:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948805.044542:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948805.044546:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948805.044547:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000075be34c1. 02000000:00000001:3.0:1784948805.044549:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948805.044559:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 00000400:00000010:0.0:1784948805.044564:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 02000000:00000001:3.0:1784948805.044565:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948805.044622:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289653888, portal 25 00000100:00000001:0.0:1784948805.044625:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948805.044630:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470016128 : -115544239535488 : ffff96e9c2a58a80) 00000400:00000010:3.0:1784948805.044635:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b69fa69e. 00000100:00000040:0.0:1784948805.044639:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948805.044645:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000200:3.0:1784948805.044654:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289654016, portal 10 00000100:00000001:0.0:1784948805.044657:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948805.044660:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948805.044665:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470022400 : -115544239529216 : ffff96e9c2a5a300) 00000100:00000040:0.0:1784948805.044666:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948805.044675:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000200:0.0:1784948805.044682:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289653888, offset 0 00000400:00000200:0.0:1784948805.044689:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:3.0:1784948805.044693:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948805.044710:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784948805.044725:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784948805.044726:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436280 00000100:00000040:3.0:1784948805.044737:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948805.044738:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x79c5 [2] + 0 00000400:00000010:0.0:1784948805.044746:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d163afc3. 00000400:00000010:3.0:1784948805.044748:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000400:00000200:0.0:1784948805.044752:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f27a392e 00000100:00000200:3.0:1784948805.044760:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289654016, offset 0 00000400:00000200:0.0:1784948805.044763:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948805.044773:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948805.044780:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948805.044786:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f27a392e 00000400:00000010:0.0:1784948805.044791:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f27a392e. 00000400:00000200:3.0:1784948805.044795:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784948805.044803:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:3.0:1784948805.044804:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:0.0:1784948805.044809:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1784948805.044813:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:0.0:1784948805.044816:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000e78df062 x1871651289653888 msgsize 224 00000400:00000200:3.0:1784948805.044820:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948805.044829:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00100000:0.0:1784948805.044831:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784948805.044837:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3268:3268:241:4294967295] 192.168.202.137@tcp LPNI seq info [2112:2112:8:4294967295] 00000100:00000040:0.0:1784948805.044839:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948805.044855:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948805.044863:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000200:3.0:1784948805.044863:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948805.044878:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000400:00000200:0.0:1784948805.044882:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000800:00000200:3.0:1784948805.044889:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:0.0:1784948805.044890:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784948805.044899:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784948805.044947:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000200:0.0:1784948805.044950:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948805.044955:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948805.044956:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948805.044963:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289653888 02000000:00000001:1.0:1784948805.044969:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948805.044973:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948805.044977:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948805.044978:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948805.044988:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2052 usecs (3 jiffies) 02000000:00000001:1.0:1784948805.044989:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.044990:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948805.044997:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948805.044998:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305744388 00000100:00100000:1.0:1784948805.045001:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289653888 00000020:00000001:1.0:1784948805.045009:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000010:3.0:1784948805.045010:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:0.0:1784948805.045016:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948805.045016:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948805.045022:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784948805.045022:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948805.045023:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000040:1.0:1784948805.045029:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948805.045031:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.045033:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305744388 00000020:00000001:1.0:1784948805.045036:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948805.045038:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305744427 00000100:00000010:3.0:1784948805.045041:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000001:1.0:1784948805.045044:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948805.045048:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784948805.045051:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305744388 00000100:00000001:1.0:1784948805.045055:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948805.045058:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305744388 00000100:00000001:0.0:1784948805.045062:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305744427 00000020:00000010:1.0:1784948805.045063:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001690d9bc. 00000100:00000010:3.0:1784948805.045064:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:1.0:1784948805.045072:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948805.045074:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948805.045076:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305744388 00000020:00000010:1.0:1784948805.045083:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001690d9bc. 00000100:00000001:3.0:1784948805.045087:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305744388 00000100:00000001:0.0:1784948805.045088:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305744427 00000020:00000010:1.0:1784948805.045093:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a17860fa. 00000100:00000010:3.0:1784948805.045096:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:0.0:1784948805.045099:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305744388 00000020:00000010:1.0:1784948805.045103:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006df70064. 00000100:00000001:3.0:1784948805.045103:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948805.045109:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305744427 00000100:00000001:3.0:1784948805.045112:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948805.045114:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:0.0:1784948805.045115:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305744388 00000100:00000001:3.0:1784948805.045117:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.045124:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.045125:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948805.045128:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948805.045131:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948805.045133:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.045139:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948805.045139:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948805.045142:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948805.045143:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948805.045146:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.045150:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948805.045153:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:2.0:1784948805.045157:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948805.045158:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948805.045159:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.045164:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000010:2.0:1784948805.045166:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000001:0.0:1784948805.045168:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948805.045168:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948805.045178:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948805.045179:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948805.045180:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948805.045183:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948805.045194:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948805.045203:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948805.045208:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000800:00000001:3.0:1784948805.045211:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948805.045213:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:1.0:1784948805.045216:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948805.045221:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948805.045224:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784948805.045227:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784948805.045234:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948805.045237:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 708 00000100:00000001:0.0:1784948805.045240:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948805.045247:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784948805.045248:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436300 00000100:00000040:1.0:1784948805.045250:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948805.045253:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948805.045261:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782156880 : -115538927394736 : ffff96eaff466050) 00000100:00000001:0.0:1784948805.045263:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948805.045272:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 154256 00000100:00000001:2.0:1784948805.045274:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948805.045280:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e78df062 x1871651289653888/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:466/0 lens 224/0 e 0 to 0 dl 1784948816 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948805.045285:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784948805.045288:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948805.045295:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948805.045309:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948805.045324:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948805.045337:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.045339:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948805.045342:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948805.045343:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:3.0:1784948805.045345:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000010:1.0:1784948805.045353:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001690d9bc. 00000100:00000010:3.0:1784948805.045354:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000092889a66. 00000100:00000040:3.0:1784948805.045361:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000092889a66 x1871651289654016 msgsize 224 00000100:00000001:1.0:1784948805.045363:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784948805.045371:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000010:1.0:1784948805.045381:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001690d9bc. 00000100:00100000:1.0:1784948805.045390:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e78df062 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289653888:12345-0@lo:400: 00000100:00000001:3.0:1784948805.045397:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:1.0:1784948805.045405:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289653888 00000800:00000001:3.0:1784948805.045408:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948805.045412:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:3.0:1784948805.045415:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948805.045420:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948805.045426:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948805.045430:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948805.045434:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948805.045440:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948805.045443:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948805.045447:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948805.045450:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948805.045454:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948805.045457:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948805.045460:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948805.045461:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00100000:0.0:1784948805.045463:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289654016 02000000:00000001:1.0:1784948805.045465:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:0.0:1784948805.045466:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948805.045469:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000010:1.0:1784948805.045471:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000afc15cec. 00000100:00000001:0.0:1784948805.045472:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948805.045476:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948805.045476:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948805.045479:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948805.045480:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289654016 00000020:00000001:0.0:1784948805.045483:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:1.0:1784948805.045483:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948805.045485:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948805.045488:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00010000:00000040:1.0:1784948805.045488:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289653888 00000020:00000040:0.0:1784948805.045492:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000001:1.0:1784948805.045492:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:0.0:1784948805.045495:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948805.045499:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000200:1.0:1784948805.045499:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e78df062 x1871651289653888/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:466/0 lens 224/224 e 0 to 0 dl 1784948816 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948805.045504:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948805.045508:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:0.0:1784948805.045512:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1784948805.045513:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:0.0:1784948805.045516:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00010000:00000001:1.0:1784948805.045516:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948805.045521:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=127 v=5 (1 1 1 1) 00000020:00000010:0.0:1784948805.045522:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000020:00000010:0.0:1784948805.045527:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000080a10e75. 00000100:00000001:1.0:1784948805.045528:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:0.0:1784948805.045532:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:1.0:1784948805.045532:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948805.045536:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948805.045539:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:0.0:1784948805.045541:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:1.0:1784948805.045542:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948805.045544:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948805.045546:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948805.045547:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948805.045550:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948805.045550:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948805.045555:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000100:00000001:0.0:1784948805.045558:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:1.0:1784948805.045560:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289653888, offset 224 00000100:00000001:0.0:1784948805.045561:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948805.045563:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948805.045567:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948805.045611:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948805.045611:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784948805.045622:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436280 00000100:00000001:0.0:1784948805.045626:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948805.045629:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784948805.045631:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x83ed [1] + 224 00000100:00100000:0.0:1784948805.045638:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 684 00000400:00000200:1.0:1784948805.045641:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784948805.045644:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000200:1.0:1784948805.045647:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948805.045648:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482529664 : -115544227021952 : ffff96e9c3647b80) 00000100:00000001:1.0:1784948805.045654:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784948805.045655:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000092889a66 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:466/0 lens 224/0 e 0 to 0 dl 1784948816 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948805.045661:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948805.045671:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784948805.045675:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948805.045676:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:0.0:1784948805.045679:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948805.045683:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00100000:0.0:1784948805.045689:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000092889a66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289654016:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948805.045695:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289654016 00000020:00000001:0.0:1784948805.045699:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948805.045703:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784948805.045704:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1784948805.045705:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948805.045709:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948805.045712:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:1.0:1784948805.045712:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:1.0:1784948805.045715:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000020:00000001:0.0:1784948805.045716:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948805.045721:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948805.045722:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:0.0:1784948805.045725:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948805.045725:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:0.0:1784948805.045728:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948805.045731:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948805.045732:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948805.045735:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948805.045737:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948805.045738:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948805.045740:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948805.045742:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784948805.045745:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00010000:00000001:1.0:1784948805.045745:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948805.045750:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784948805.045751:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:0.0:1784948805.045753:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000eea3db3f. 00000020:00000001:1.0:1784948805.045754:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:2.0:1784948805.045756:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1784948805.045759:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948805.045761:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000075be34c1. 00000100:00000040:1.0:1784948805.045764:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e78df062 x1871651289653888/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:466/0 lens 224/224 e 0 to 0 dl 1784948816 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948805.045766:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948805.045770:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000020:00000001:0.0:1784948805.045772:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948805.045777:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00010000:00000040:0.0:1784948805.045780:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289654016 00000100:00000001:2.0:1784948805.045787:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:0.0:1784948805.045788:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00100000:1.0:1784948805.045793:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e78df062 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289653888:12345-0@lo:400: Request processed in 448us (971us total) trans 0 rc 0/0 00000100:00000200:2.0:1784948805.045800:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:0.0:1784948805.045801:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000092889a66 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:466/0 lens 224/224 e 0 to 0 dl 1784948816 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948805.045813:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 708 00000100:00000040:1.0:1784948805.045825:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948805.045833:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948805.045839:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948805.045847:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006df70064. 00010000:00000001:0.0:1784948805.045854:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000200:2.0:1784948805.045855:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948805.045856:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00010000:00000001:0.0:1784948805.045861:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948805.045865:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a17860fa. 00000100:00001000:0.0:1784948805.045868:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=122 v=5 (1 1 1 1) 00000020:00000040:1.0:1784948805.045893:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948805.045908:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784948805.045916:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948805.045917:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1784948805.045918:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 02000000:00000010:1.0:1784948805.045961:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000afc15cec. 00000100:00000001:0.0:1784948805.045967:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948805.045968:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:0.0:1784948805.045972:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784948805.045973:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948805.045976:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948805.045979:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948805.045981:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948805.045984:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784948805.045988:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948805.045990:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:1.0:1784948805.045991:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b2f60a65. 00000100:00000001:2.0:1784948805.045997:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:1.0:1784948805.046004:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000026a1c0e3. 00000100:00000001:2.0:1784948805.046005:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948805.046006:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948805.046016:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948805.046018:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00001000:2.0:1784948805.046024:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=127 v=5 (5 5 5 5) 00000100:00000200:0.0:1784948805.046029:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289654016, offset 224 00000100:00001000:2.0:1784948805.046034:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=127 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948805.046042:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948805.046047:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948805.046050:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00010000:00000001:2.0:1784948805.046052:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784948805.046060:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948805.046064:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948805.046072:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948805.046078:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948805.046097:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948805.046103:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3269:3269:241:4294967295] 192.168.202.137@tcp LPNI seq info [2113:2113:8:4294967295] 00000100:00000001:2.0:1784948805.046105:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948805.046111:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948805.046120:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948805.046140:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:2.0:1784948805.046156:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000098475f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289653888:0@lo:400: 00000800:00000200:0.0:1784948805.046165:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948805.046169:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948805.046175:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000086e4d21. 00000100:00000001:2.0:1784948805.046176:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948805.046184:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000098475f18 x1871651289653888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948805.046194:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784948805.046203:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948805.046209:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:0.0:1784948805.046215:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000010:2.0:1784948805.046216:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 02000000:00000001:2.0:1784948805.046229:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948805.046235:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000800:00000200:0.0:1784948805.046236:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000086e4d21 type 1, nob 320 niov 1 nkiov 1 00000020:00000040:2.0:1784948805.046239:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784948805.046248:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948805.046251:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c03b4f3a. 02000000:00000001:2.0:1784948805.046256:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948805.046258:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948805.046260:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000098475f18. 00000100:00000001:2.0:1784948805.046264:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948805.046266:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948805.046267:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948805.046277:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948805.046281:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948805.046286:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1784948805.046288:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:2.0:1784948805.046294:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948805.046295:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948805.046302:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784948805.046303:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784948805.046308:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948805.046316:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000092889a66 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:466/0 lens 224/224 e 0 to 0 dl 1784948816 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948805.046331:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000092889a66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289654016:12345-192.168.202.137@tcp:400: Request processed in 657us (966us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948805.046344:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 684 00000100:00000040:0.0:1784948805.046348:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948805.046362:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948805.046364:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948805.046368:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000080a10e75. 00000020:00000010:0.0:1784948805.046373:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948805.046378:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000020:00000040:0.0:1784948805.046383:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948805.046387:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948805.046441:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948805.046448:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000086e4d21. 00000400:00000200:3.0:1784948805.046455:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948805.046467:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948805.046474:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948805.046482:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948805.046483:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:3.0:1784948805.046490:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000800:00000001:2.0:1784948805.046492:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948805.046497:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784948805.046500:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948805.046501:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948805.046516:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948805.046532:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436300 00000400:00000200:2.0:1784948805.046540:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x83f1 [1] + 224 00000800:00000001:2.0:1784948805.046547:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948805.046561:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948805.046563:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948805.046618:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948805.046623:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948805.046627:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948805.046637:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948805.046656:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948805.046662:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948805.046665:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948805.046700:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948805.046707:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948805.046712:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948805.046715:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948805.046719:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b69fa69e. 00000400:00000200:3.0:1784948805.046727:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948805.046729:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000001:3.0:1784948805.046733:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948805.046741:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948805.046757:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948805.046769:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948805.046771:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948805.046775:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948805.046778:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948805.046780:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948805.046781:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948805.046784:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948805.046789:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=122 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948805.046793:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=122 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948805.046795:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948805.046798:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948805.046800:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948805.046804:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948805.046812:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.046815:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948805.046817:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948805.046819:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948805.046820:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948805.046823:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948805.046830:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948805.046832:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948805.046841:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948805.046844:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948805.046852:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289654016:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948805.046856:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948805.046858:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948805.046860:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b8226094 x1871651289654016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948805.046867:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948805.046868:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948805.046870:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948805.046874:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948805.046875:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948805.046877:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948805.046879:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948805.046881:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000762a0fe3. 02000000:00000001:3.0:1784948805.046883:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948805.046884:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948805.046886:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b8226094. 00000100:00000001:3.0:1784948805.046889:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948805.046891:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948805.046892:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948805.046896:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948805.046932:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948805.046935:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948805.046937:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948805.088009:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305744427, at 4305744428 00000100:00000040:3.0:1784948805.088021:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305744427, at 4305744428 00000100:00000040:2.0:1784948805.088023:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305744427, at 4305744428 00000100:00000010:1.0:1784948805.088027:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:3.0:1784948805.088040:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:2.0:1784948805.088043:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:0.0:1784948805.088065:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305744427, at 4305744428 00000100:00000010:0.0:1784948805.088082:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000400:00000200:1.0:1784948805.236037:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948805.236047:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a2236612. 00000400:00000010:1.0:1784948805.236061:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a17860fa. 00000400:00000010:1.0:1784948805.236068:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000400:00000010:1.0:1784948805.236077:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008015f6b2. 00000400:00000200:1.0:1784948805.236083:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948805.236100:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948805.236109:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948805.236114:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3270:3270:241:4294967295] 192.168.202.137@tcp LPNI seq info [2114:2114:8:4294967295] 00000400:00000200:1.0:1784948805.236120:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948805.236128:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948805.236134:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948805.236140:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000800:00000200:1.0:1784948805.236147:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948805.236155:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948805.236160:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948805.236194:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11079 health :900 00000800:00000200:3.0:1784948805.236342:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948805.236350:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:3.0:1784948805.236360:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948805.236373:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784948805.236382:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784948805.236385:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948805.236388:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784948805.236401:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948805.236407:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948805.236414:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948805.236424:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948805.236439:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948805.236449:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948805.236458:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948805.236463:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3271:3271:241:4294967295] 192.168.202.137@tcp LPNI seq info [2115:2115:8:4294967295] 00000400:00000200:0.0:1784948805.236473:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948805.236478:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948805.236483:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b51ad697. 00000800:00000200:0.0:1784948805.236488:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948805.236493:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948805.236496:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b51ad697 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784948805.236600:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948805.236653:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948805.236658:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b51ad697. 00000400:00000200:0.0:1784948805.236663:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948805.236669:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948805.236675:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948805.236690:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948805.236696:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948805.236697:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948805.236706:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948805.236716:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948805.236730:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8409 00000800:00000001:3.0:1784948805.236735:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948805.236743:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948805.236745:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948805.236749:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948805.236755:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:3.0:1784948805.236758:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000400:00000200:3.0:1784948805.236761:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948805.236763:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a2236612. 00000400:00000010:3.0:1784948805.236767:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a17860fa. 00000800:00000001:3.0:1784948805.236774:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948805.236776:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948805.876017:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948805.876023:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948805.876028:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948805.876029:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948805.876113:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948805.876117:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948805.876120:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948806.067984:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948806.067995:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948806.067997:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948806.068003:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948806.068005:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948806.068008:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948806.068010:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948806.900039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948806.900047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948806.900053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948806.900055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948806.900119:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948806.900124:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948806.900127:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948807.092033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948807.092053:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948807.092056:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948807.092064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948807.092068:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948807.092072:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948807.092076:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948807.924052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948807.924073:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948807.924082:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948807.924085:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948807.924202:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948807.924209:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948807.924214:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948808.116072:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948808.116090:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948808.116093:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948808.116100:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948808.116104:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948808.116109:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948808.116113:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948808.948032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948808.948040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948808.948055:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948808.948058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948808.948128:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948808.948133:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948808.948137:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948809.140218:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948809.140237:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948809.140242:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948809.140252:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948809.140258:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948809.140264:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948809.140269:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948809.972059:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948809.972069:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948809.972077:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948809.972081:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948809.972085:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948809.972089:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948809.972094:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948809.972097:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948809.972100:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948809.972103:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948809.972188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948809.972197:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948809.972203:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948810.163972:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948810.163983:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948810.163984:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948810.163988:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948810.163990:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948810.163993:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948810.163995:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.164023:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948810.164039:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948810.164045:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000020:00000040:0.0:1784948810.164050:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948810.164054:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948810.164057:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948810.164061:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948810.164066:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b51ad697. 00000100:00000001:0.0:1784948810.164072:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.164080:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948810.164089:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948810.164090:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.164094:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_00_03+3] req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948810.164114:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948810.164117:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948810.164120:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948810.164123:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948810.164127:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948810.164131:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ee0ebd63. 00000020:00000040:0.0:1784948810.164134:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948810.164136:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948810.164137:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948810.164140:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784948810.164141:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948810.164143:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784948810.164145:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948810.164150:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948810.164151:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000eafb2cdc. 00000100:00000001:1.0:1784948810.164152:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948810.164157:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948810.164157:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164164:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948810.164166:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.164167:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1784948810.164170:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948810.164173:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948810.164178:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948810.164183:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289654144:0@lo:400: 00000100:00000001:0.0:1784948810.164187:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948810.164188:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948810.164190:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948810.164194:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948810.164196:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.164197:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0ebd63] to pc [ptlrpcd_00_00+0] req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948810.164199:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948810.164206:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948810.164209:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948810.164221:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b69fa69e. 00000100:00000001:0.0:1784948810.164224:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948810.164228:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000040:0.0:1784948810.164231:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11087) 00000100:00000200:3.0:1784948810.164235:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289654144, portal 25 00000100:00000001:3.0:1784948810.164238:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948810.164240:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000001:1.0:1784948810.164243:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948810.164244:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948810.164249:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948810.164251:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948810.164253:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948810.164256:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164256:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948810.164262:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948810.164262:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948810.164266:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 00000100:00000200:3.0:1784948810.164270:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289654144, offset 0 00000100:00000001:1.0:1784948810.164279:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1784948810.164279:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948810.164281:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948810.164284:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948810.164285:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948810.164287:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948810.164290:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ee0ebd63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289654272:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784948810.164293:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784948810.164297:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948810.164300:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948810.164301:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948810.164303:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948810.164306:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436380 02000000:00000010:1.0:1784948810.164307:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001e170cf1. 02000000:00000001:1.0:1784948810.164311:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948810.164313:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x79e1 [2] + 0 00000400:00000010:1.0:1784948810.164314:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006be72eea. 00000400:00000010:3.0:1784948810.164318:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fed0cd52. 00000400:00000200:3.0:1784948810.164324:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bf496567 00000400:00000010:1.0:1784948810.164328:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000200:3.0:1784948810.164330:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1784948810.164333:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289654272, portal 10 00000400:00000200:3.0:1784948810.164336:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948810.164337:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948810.164340:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558293120 : -115544151258496 : ffff96e9c7e88a80) 00000400:00000200:3.0:1784948810.164343:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bf496567 00000400:00000010:3.0:1784948810.164346:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bf496567. 00000100:00000040:1.0:1784948810.164347:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948810.164350:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948810.164353:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784948810.164358:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948810.164358:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000009e6f08df x1871651289654144 msgsize 224 00000100:00000040:1.0:1784948810.164364:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948810.164366:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000200:1.0:1784948810.164368:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289654272, offset 0 00000100:00100000:3.0:1784948810.164372:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:1.0:1784948810.164376:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784948810.164377:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784948810.164385:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784948810.164388:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948810.164391:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948810.164395:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948810.164399:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948810.164402:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948810.164406:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3272:3272:241:4294967295] 192.168.202.137@tcp LPNI seq info [2116:2116:8:4294967295] 00000400:00000200:1.0:1784948810.164414:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784948810.164414:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948810.164418:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289654144 00000800:00000200:1.0:1784948810.164422:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948810.164422:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948810.164425:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:3.0:1784948810.164426:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000800:00000010:1.0:1784948810.164428:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000100:00000001:2.0:1784948810.164429:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948810.164429:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 02000000:00000001:2.0:1784948810.164436:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164436:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:1.0:1784948810.164438:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:2.0:1784948810.164441:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289654144 00000100:00000200:3.0:1784948810.164442:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948810.164445:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948810.164448:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000200:1.0:1784948810.164451:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:2.0:1784948810.164453:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948810.164457:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000800:00000200:1.0:1784948810.164458:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948810.164459:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784948810.164462:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:3.0:1784948810.164465:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948810.164466:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948810.164472:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948810.164474:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164476:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948810.164478:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:3.0:1784948810.164479:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784948810.164481:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2053 usecs (3 jiffies) 00000100:00000001:2.0:1784948810.164484:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948810.164487:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000001:3.0:1784948810.164487:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948810.164488:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305749507 00000100:00000001:3.0:1784948810.164493:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305749546 00000020:00000010:2.0:1784948810.164496:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f1904ca. 00000100:00000010:1.0:1784948810.164497:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000092889e7d. 00000020:00000010:2.0:1784948810.164503:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000ced696a. 00000100:00000001:1.0:1784948810.164507:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000010:2.0:1784948810.164508:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000001:1.0:1784948810.164516:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305749507 00000100:00000040:2.0:1784948810.164518:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948810.164519:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305749507 00000100:00000001:2.0:1784948810.164523:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:1.0:1784948810.164524:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000c54be64. 00000100:00000001:3.0:1784948810.164526:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305749546 00000100:00000001:2.0:1784948810.164543:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948810.164546:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948810.164550:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948810.164556:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305749507 00000100:00000001:2.0:1784948810.164556:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164557:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305749507 00000100:00000010:1.0:1784948810.164564:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:3.0:1784948810.164564:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305749546 00000100:00000001:2.0:1784948810.164568:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948810.164572:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948810.164577:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948810.164580:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305749507 00000100:00000001:2.0:1784948810.164580:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948810.164581:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305749507 00000100:00100000:2.0:1784948810.164587:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 709 00000100:00000010:1.0:1784948810.164588:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000001:3.0:1784948810.164589:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305749546 00000100:00000040:2.0:1784948810.164592:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948810.164595:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948810.164596:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782153808 : -115538927397808 : ffff96eaff465450) 00000100:00000040:2.0:1784948810.164603:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009e6f08df x1871651289654144/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:471/0 lens 224/0 e 0 to 0 dl 1784948821 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948810.164604:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948810.164605:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305749507 00000100:00000001:1.0:1784948810.164612:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164613:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948810.164615:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948810.164620:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164620:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948810.164621:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:3.0:1784948810.164624:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948810.164625:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948810.164626:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948810.164627:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948810.164628:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948810.164632:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000020:00000010:2.0:1784948810.164632:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000001:1.0:1784948810.164637:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948810.164637:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1784948810.164638:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009e6f08df pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289654144:12345-0@lo:400: 00000400:00000200:0.0:1784948810.164640:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948810.164640:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948810.164642:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000200:2.0:1784948810.164648:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289654144 00000100:00000001:1.0:1784948810.164650:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948810.164651:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948810.164652:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784948810.164654:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948810.164655:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948810.164655:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948810.164656:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784948810.164660:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948810.164660:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164660:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948810.164663:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000020:00000001:2.0:1784948810.164667:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:0.0:1784948810.164668:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000100:00000001:3.0:1784948810.164668:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948810.164669:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948810.164670:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784948810.164676:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1784948810.164677:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948810.164680:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948810.164683:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:0.0:1784948810.164685:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948810.164686:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948810.164689:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948810.164693:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948810.164696:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948810.164699:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:1.0:1784948810.164703:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948810.164704:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1784948810.164705:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000ebcf220. 02000000:00000001:2.0:1784948810.164708:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.164709:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948810.164710:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948810.164714:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948810.164714:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948810.164719:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289654144 00000800:00000001:1.0:1784948810.164720:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948810.164723:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948810.164725:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948810.164725:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948810.164727:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000200:2.0:1784948810.164731:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009e6f08df x1871651289654144/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:471/0 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948810.164738:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:2.0:1784948810.164742:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948810.164745:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948810.164747:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=132 v=5 (1 1 1 1) 00000400:00000200:1.0:1784948810.164752:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436400 00000100:00000001:2.0:1784948810.164753:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948810.164756:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948810.164760:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784948810.164762:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948810.164763:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948810.164767:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 154480 00000100:00000001:2.0:1784948810.164767:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948810.164773:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948810.164778:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000800:00000001:1.0:1784948810.164779:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:2.0:1784948810.164782:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289654144, offset 224 00000800:00000001:1.0:1784948810.164787:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948810.164788:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948810.164793:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948810.164800:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948810.164801:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784948810.164808:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948810.164811:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1784948810.164814:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436380 00000100:00000010:1.0:1784948810.164817:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005e9e18c4. 00000100:00000040:1.0:1784948810.164821:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000005e9e18c4 x1871651289654272 msgsize 224 00000400:00000200:2.0:1784948810.164824:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x840d [1] + 224 00000100:00100000:1.0:1784948810.164829:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784948810.164837:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948810.164841:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948810.164843:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784948810.164846:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:1.0:1784948810.164851:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:2.0:1784948810.164852:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948810.164867:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948810.164876:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948810.164890:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948810.164890:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:0.0:1784948810.164894:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289654272 00000400:00000200:2.0:1784948810.164896:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 02000000:00000001:0.0:1784948810.164897:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000010:2.0:1784948810.164899:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:0.0:1784948810.164900:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948810.164902:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948810.164927:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784948810.164930:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948810.164931:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.164933:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948810.164935:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948810.164936:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:3.0:1784948810.164940:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000001:2.0:1784948810.164941:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 02000000:00000001:0.0:1784948810.164942:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784948810.164943:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1784948810.164944:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948810.164948:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:3.0:1784948810.164948:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b69fa69e. 00000100:00100000:0.0:1784948810.164949:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289654272 00000100:00000040:2.0:1784948810.164953:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009e6f08df x1871651289654144/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:471/0 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948810.164955:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:3.0:1784948810.164956:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000020:00000040:0.0:1784948810.164960:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:3.0:1784948810.164960:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000020:00000001:0.0:1784948810.164966:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:2.0:1784948810.164967:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009e6f08df pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289654144:12345-0@lo:400: Request processed in 349us (606us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948810.164967:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:0.0:1784948810.164972:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:3.0:1784948810.164974:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948810.164978:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 709 00000020:00000001:0.0:1784948810.164980:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:2.0:1784948810.164984:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:0.0:1784948810.164987:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948810.164999:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948810.165001:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:3.0:1784948810.165002:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948810.165003:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948810.165005:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000874e7bc2. 00000020:00000010:2.0:1784948810.165008:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000ced696a. 00000100:00000001:0.0:1784948810.165010:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948810.165015:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:0.0:1784948810.165016:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000874e7bc2. 00000100:00000001:3.0:1784948810.165018:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784948810.165020:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f1904ca. 00000100:00000001:3.0:1784948810.165022:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948810.165024:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000100:00000001:3.0:1784948810.165025:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000040:2.0:1784948810.165026:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:3.0:1784948810.165029:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:0.0:1784948810.165031:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000080a10e75. 00000100:00000001:3.0:1784948810.165031:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:2.0:1784948810.165032:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948810.165035:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948810.165037:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000000ebcf220. 00000020:00000010:0.0:1784948810.165038:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 02000000:00000001:3.0:1784948810.165041:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948810.165042:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784948810.165045:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784948810.165048:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=132 v=5 (5 5 5 5) 00000100:00000040:0.0:1784948810.165049:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000200:2.0:1784948810.165050:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948810.165053:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000010:2.0:1784948810.165056:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba88f347. 00000100:00000001:0.0:1784948810.165057:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00001000:3.0:1784948810.165058:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=132 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948810.165061:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948810.165062:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005c60caa0. 00000100:00000001:3.0:1784948810.165064:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948810.165068:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948810.165070:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948810.165073:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00010000:00000001:3.0:1784948810.165073:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784948810.165078:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948810.165078:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.165085:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948810.165089:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948810.165091:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948810.165100:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948810.165103:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948810.165104:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948810.165108:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784948810.165108:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948810.165112:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948810.165117:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289654144:0@lo:400: 00000100:00100000:0.0:1784948810.165123:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 685 00000100:00000001:1.0:1784948810.165123:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948810.165124:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948810.165127:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948810.165127:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948810.165128:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948810.165131:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948810.165132:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529451652352 : -115544257899264 : ffff96e9c18d5500) 00000100:00000040:3.0:1784948810.165134:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b3c18d2b x1871651289654144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948810.165135:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.165139:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005e9e18c4 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:471/0 lens 224/0 e 0 to 0 dl 1784948821 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948810.165149:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948810.165152:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948810.165155:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 00000100:00000001:0.0:1784948810.165158:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:3.0:1784948810.165159:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948810.165161:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:0.0:1784948810.165162:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000874e7bc2. 00000020:00000040:3.0:1784948810.165163:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948810.165169:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948810.165169:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948810.165172:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b51ad697. 00000020:00000010:0.0:1784948810.165173:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000874e7bc2. 02000000:00000001:3.0:1784948810.165176:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948810.165178:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948810.165180:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b3c18d2b. 00000100:00100000:0.0:1784948810.165181:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005e9e18c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289654272:12345-192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948810.165184:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948810.165186:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948810.165188:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289654272 00000100:00000001:3.0:1784948810.165189:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948810.165193:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784948810.165194:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784948810.165197:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784948810.165197:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.165199:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784948810.165201:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948810.165202:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948810.165207:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948810.165210:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948810.165213:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948810.165215:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948810.165217:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948810.165218:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948810.165219:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948810.165221:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948810.165223:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948810.165224:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948810.165227:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008046922a. 02000000:00000001:0.0:1784948810.165228:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948810.165230:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948810.165231:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948810.165232:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289654272 00010000:00000001:0.0:1784948810.165234:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948810.165236:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005e9e18c4 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:471/0 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948810.165241:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948810.165243:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948810.165244:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=127 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948810.165247:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948810.165248:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948810.165249:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948810.165251:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948810.165252:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948810.165254:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948810.165256:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948810.165257:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000100:00000200:0.0:1784948810.165260:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289654272, offset 224 00000400:00000200:0.0:1784948810.165263:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948810.165268:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948810.165271:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3273:3273:241:4294967295] 192.168.202.137@tcp LPNI seq info [2117:2117:8:4294967295] 00000400:00000200:0.0:1784948810.165277:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948810.165280:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948810.165282:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006487a147. 00000800:00000200:0.0:1784948810.165285:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948810.165288:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948810.165290:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000006487a147 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948810.165300:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948810.165302:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948810.165304:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948810.165305:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948810.165306:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948810.165308:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005e9e18c4 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:471/0 lens 224/224 e 0 to 0 dl 1784948821 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948810.165314:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005e9e18c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289654272:12345-192.168.202.137@tcp:400: Request processed in 156us (492us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948810.165321:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 685 00000100:00000040:0.0:1784948810.165323:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948810.165325:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948810.165326:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948810.165327:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000080a10e75. 00000020:00000010:0.0:1784948810.165330:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948810.165332:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000020:00000040:0.0:1784948810.165335:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948810.165337:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948810.165379:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948810.165380:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784948810.165384:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006487a147. 00000800:00000001:0.0:1784948810.165388:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948810.165388:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948810.165393:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948810.165397:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948810.165398:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948810.165404:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000400:00000010:3.0:1784948810.165407:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000400:00000200:0.0:1784948810.165408:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948810.165413:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948810.165415:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784948810.165418:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436400 00000400:00000200:0.0:1784948810.165423:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8415 [1] + 224 00000800:00000001:0.0:1784948810.165426:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948810.165432:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948810.165434:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948810.165436:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948810.165439:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948810.165441:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948810.165447:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948810.165458:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948810.165463:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948810.165466:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948810.165486:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948810.165489:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948810.165492:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948810.165494:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948810.165495:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006be72eea. 00000400:00000200:1.0:1784948810.165499:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784948810.165501:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784948810.165503:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948810.165507:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948810.165515:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948810.165519:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948810.165520:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948810.165521:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948810.165523:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948810.165524:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948810.165525:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948810.165540:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948810.165544:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=127 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948810.165546:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=127 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948810.165548:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948810.165549:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948810.165550:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948810.165552:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948810.165557:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948810.165559:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948810.165560:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948810.165561:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948810.165562:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948810.165564:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948810.165583:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948810.165584:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948810.165585:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948810.165587:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948810.165591:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ee0ebd63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289654272:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948810.165593:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948810.165594:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948810.165596:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ee0ebd63 x1871651289654272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948810.165599:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948810.165600:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948810.165601:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001e170cf1. 02000000:00000001:1.0:1784948810.165605:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948810.165606:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948810.165607:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948810.165608:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948810.165609:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000eafb2cdc. 02000000:00000001:1.0:1784948810.165611:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948810.165612:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948810.165612:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ee0ebd63. 00000100:00000001:1.0:1784948810.165614:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948810.165615:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948810.165616:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948810.165619:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948810.165619:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948810.165620:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948810.165622:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948810.206964:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305749546, at 4305749547 00000100:00000040:0.0:1784948810.206970:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305749546, at 4305749547 00000100:00000040:1.0:1784948810.206970:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305749546, at 4305749547 00000100:00000010:3.0:1784948810.206979:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000010:1.0:1784948810.206981:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000c54be64. 00000100:00000010:0.0:1784948810.206982:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000092889e7d. 00000100:00000040:2.0:1784948810.206987:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305749546, at 4305749547 00000100:00000010:2.0:1784948810.206997:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00010000:00000001:2.0:1784948810.995979:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948810.995985:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948810.995990:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948810.995991:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948810.996056:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948810.996061:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948810.996064:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948811.187975:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948811.187987:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948811.187988:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948811.187992:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948811.187994:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948811.187996:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948811.187998:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948812.020019:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948812.020028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948812.020037:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948812.020039:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948812.020137:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948812.020146:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948812.020154:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948812.212032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948812.212050:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948812.212053:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948812.212060:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948812.212064:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948812.212069:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948812.212072:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948813.044031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948813.044041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948813.044049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948813.044051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948813.044161:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948813.044171:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948813.044178:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948813.236032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948813.236049:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948813.236052:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948813.236059:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948813.236063:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948813.236068:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948813.236072:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948814.068015:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948814.068023:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948814.068030:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948814.068033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948814.068112:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948814.068118:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948814.068122:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948814.259994:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948814.260007:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948814.260010:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948814.260016:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948814.260020:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948814.260023:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948814.260026:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948815.092020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948815.092030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948815.092038:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948815.092040:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948815.092174:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948815.092181:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948815.092186:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948815.284008:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948815.284025:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948815.284028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948815.284035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948815.284038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948815.284040:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948815.284043:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.284052:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948815.284064:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948815.284075:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000c8b06f4. 00000020:00000040:0.0:1784948815.284084:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948815.284090:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948815.284095:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948815.284102:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948815.284111:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000008746b1e6. 00000100:00000001:0.0:1784948815.284120:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948815.284131:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948815.284146:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948815.284148:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948815.284155:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c8b06f4] to pc [ptlrpcd_00_01+1] req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948815.284192:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948815.284195:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948815.284201:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948815.284205:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948815.284210:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948815.284213:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 00000020:00000040:0.0:1784948815.284216:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948815.284219:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948815.284221:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948815.284223:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948815.284224:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.284229:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:0.0:1784948815.284232:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002aeee8e9. 00000100:00000001:2.0:1784948815.284236:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948815.284239:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948815.284248:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948815.284254:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948815.284271:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948815.284282:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.284298:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948815.284300:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948815.284303:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948815.284305:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948815.284311:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948815.284316:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59e6b47] to pc [ptlrpcd_00_02+2] req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948815.284319:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948815.284328:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948815.284339:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000c8b06f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289654400:0@lo:400: 00000100:00000001:2.0:1784948815.284350:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948815.284357:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948815.284360:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948815.284364:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948815.284365:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948815.284372:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000f1904ca. 00000100:00000040:0.0:1784948815.284374:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11092) 02000000:00000001:2.0:1784948815.284380:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948815.284392:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000068d5a5a2. 00000100:00000001:1.0:1784948815.284395:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948815.284402:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1784948815.284402:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000001:1.0:1784948815.284406:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000200:2.0:1784948815.284414:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289654400, portal 25 00000100:00000040:1.0:1784948815.284422:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948815.284423:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948815.284429:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558297600 : -115544151254016 : ffff96e9c7e89c00) 00000100:00000001:3.0:1784948815.284436:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948815.284445:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948815.284446:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948815.284462:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948815.284468:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.284471:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784948815.284476:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1784948815.284478:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784948815.284482:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948815.284517:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948815.284522:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289654528:192.168.202.137@tcp:400: 00000400:00000010:2.0:1784948815.284524:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000001:1.0:1784948815.284534:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000200:2.0:1784948815.284536:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289654400, offset 0 02000000:00000001:1.0:1784948815.284541:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948815.284547:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948815.284554:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784948815.284555:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948815.284567:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dbce179e. 02000000:00000001:1.0:1784948815.284570:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948815.284578:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006be72eea. 00000400:00000200:2.0:1784948815.284579:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:1.0:1784948815.284594:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00000200:1.0:1784948815.284604:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289654528, portal 10 00000400:00000200:2.0:1784948815.284606:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436480 00000100:00000001:1.0:1784948815.284612:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948815.284617:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558303872 : -115544151247744 : ffff96e9c7e8b480) 00000400:00000200:2.0:1784948815.284623:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7a09 [2] + 0 00000400:00000010:2.0:1784948815.284631:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000014751922. 00000100:00000040:1.0:1784948815.284635:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948815.284642:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000003c50331a 00000100:00000001:1.0:1784948815.284653:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784948815.284657:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784948815.284659:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948815.284666:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000200:2.0:1784948815.284669:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:1.0:1784948815.284675:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289654528, offset 0 00000400:00000200:2.0:1784948815.284680:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003c50331a 00000400:00000010:2.0:1784948815.284700:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000003c50331a. 00000100:00000001:2.0:1784948815.284718:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948815.284724:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:1.0:1784948815.284725:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784948815.284736:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f7830c1e x1871651289654400 msgsize 224 00000100:00100000:2.0:1784948815.284746:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:1.0:1784948815.284751:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:2.0:1784948815.284753:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784948815.284764:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948815.284771:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948815.284775:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784948815.284778:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948815.284780:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948815.284788:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3274:3274:241:4294967295] 192.168.202.137@tcp LPNI seq info [2118:2118:8:4294967295] 00000400:00000200:1.0:1784948815.284803:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948815.284810:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948815.284816:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000800:00000200:1.0:1784948815.284824:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784948815.284829:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784948815.284835:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000800:00000200:1.0:1784948815.284842:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784948815.284845:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:1.0:1784948815.284853:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:2.0:1784948815.284857:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948815.284875:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948815.284879:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948815.284887:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948815.284891:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948815.284900:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289654400 00000100:00000040:1.0:1784948815.284902:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2053 usecs (3 jiffies) 00000100:00000001:2.0:1784948815.284951:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948815.284957:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948815.284959:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305754628 00000100:00000001:2.0:1784948815.284959:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948815.284966:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948815.284972:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948815.284973:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:3.0:1784948815.284973:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.284985:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305754667 02000000:00000001:3.0:1784948815.284988:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948815.284992:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00100000:3.0:1784948815.284999:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289654400 00000020:00000001:3.0:1784948815.285006:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948815.285007:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305754628 00000020:00000040:3.0:1784948815.285010:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784948815.285012:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305754628 00000100:00000010:1.0:1784948815.285018:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000020:00000001:3.0:1784948815.285019:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948815.285024:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305754667 00000020:00000040:3.0:1784948815.285028:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948815.285031:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000001:3.0:1784948815.285038:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948815.285045:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305754628 00000020:00000001:3.0:1784948815.285045:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948815.285047:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305754628 00000100:00000001:3.0:1784948815.285056:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948815.285066:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305754667 00000020:00000010:3.0:1784948815.285067:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d30e1e5e. 00000100:00000010:1.0:1784948815.285068:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005975f076. 00000100:00000001:3.0:1784948815.285078:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948815.285083:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000010:3.0:1784948815.285085:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d30e1e5e. 00000800:00000200:0.0:1784948815.285092:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948815.285099:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000100:00000001:1.0:1784948815.285101:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305754628 00000020:00000010:3.0:1784948815.285103:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eb3e5fa2. 00000100:00000001:2.0:1784948815.285107:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305754628 00000400:00000200:0.0:1784948815.285109:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784948815.285114:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003f7291f9. 00000100:00000010:1.0:1784948815.285118:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000004f2eb99f. 00000100:00000001:2.0:1784948815.285124:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305754667 00000020:00000010:3.0:1784948815.285126:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:1.0:1784948815.285130:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000400:00000200:0.0:1784948815.285133:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948815.285148:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000100:00000001:1.0:1784948815.285152:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948815.285153:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000040:3.0:1784948815.285154:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948815.285155:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305754628 00000100:00000001:3.0:1784948815.285165:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948815.285167:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948815.285168:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.285170:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948815.285179:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948815.285180:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948815.285181:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948815.285191:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948815.285192:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948815.285194:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948815.285200:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.285206:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948815.285210:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948815.285218:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948815.285219:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948815.285219:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.285226:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948815.285231:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948815.285233:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948815.285234:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948815.285236:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948815.285242:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948815.285244:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948815.285244:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948815.285245:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.285248:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948815.285262:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.285263:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948815.285270:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948815.285275:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948815.285277:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948815.285283:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948815.285295:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 710 00000800:00000001:0.0:1784948815.285296:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948815.285304:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948815.285308:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782159952 : -115538927391664 : ffff96eaff466c50) 00000800:00000001:0.0:1784948815.285313:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948815.285319:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f7830c1e x1871651289654400/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:476/0 lens 224/0 e 0 to 0 dl 1784948826 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948815.285322:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948815.285330:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948815.285344:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784948815.285351:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000010:3.0:1784948815.285351:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d30e1e5e. 00000100:00000001:3.0:1784948815.285360:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948815.285365:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d30e1e5e. 00000400:00000200:0.0:1784948815.285378:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436500 00000100:00100000:3.0:1784948815.285379:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f7830c1e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289654400:12345-0@lo:400: 00000100:00000200:3.0:1784948815.285391:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289654400 00000020:00000001:3.0:1784948815.285396:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784948815.285397:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 154704 00000020:00000001:3.0:1784948815.285402:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948815.285407:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948815.285408:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948815.285412:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948815.285416:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000001:0.0:1784948815.285420:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948815.285422:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948815.285427:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948815.285429:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784948815.285432:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948815.285437:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:0.0:1784948815.285441:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784948815.285443:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948815.285448:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948815.285451:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948815.285453:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000200:0.0:1784948815.285456:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000001:3.0:1784948815.285458:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000010:0.0:1784948815.285465:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001c81e7dd. 02000000:00000010:3.0:1784948815.285467:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 02000000:00000001:3.0:1784948815.285475:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948815.285477:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000001c81e7dd x1871651289654528 msgsize 224 00000100:00000001:3.0:1784948815.285479:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948815.285512:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948815.285518:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289654400 00010000:00000001:3.0:1784948815.285526:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00100000:0.0:1784948815.285532:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000200:3.0:1784948815.285535:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f7830c1e x1871651289654400/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:476/0 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948815.285550:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948815.285552:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948815.285556:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=137 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948815.285562:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948815.285563:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:3.0:1784948815.285568:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948815.285574:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948815.285578:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948815.285580:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948815.285581:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948815.285585:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948815.285592:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:0.0:1784948815.285594:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948815.285598:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000200:3.0:1784948815.285607:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289654400, offset 224 00000400:00000200:3.0:1784948815.285617:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948815.285622:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948815.285628:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289654528 00000400:00000200:3.0:1784948815.285633:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784948815.285635:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948815.285641:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948815.285644:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948815.285650:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436480 02000000:00000001:2.0:1784948815.285651:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948815.285662:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289654528 00000020:00000001:2.0:1784948815.285668:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:3.0:1784948815.285668:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8429 [1] + 224 00000020:00000040:2.0:1784948815.285672:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784948815.285678:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784948815.285681:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1784948815.285682:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784948815.285691:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784948815.285692:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:2.0:1784948815.285700:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948815.285705:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784948815.285708:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:3.0:1784948815.285713:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948815.285716:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:2.0:1784948815.285725:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948815.285733:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000020:00000010:2.0:1784948815.285742:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000100:00000040:3.0:1784948815.285748:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948815.285752:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000ced696a. 00000020:00000010:2.0:1784948815.285762:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000040:2.0:1784948815.285775:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948815.285778:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948815.285779:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948815.285782:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948815.285786:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948815.285790:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948815.285796:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784948815.285798:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948815.285799:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948815.285804:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000001:3.0:1784948815.285809:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948815.285811:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784948815.285816:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.285819:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948815.285821:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:3.0:1784948815.285828:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948815.285832:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948815.285835:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784948815.285836:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948815.285842:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:3.0:1784948815.285845:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f7830c1e x1871651289654400/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:476/0 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948815.285853:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784948815.285856:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 686 00000100:00000001:0.0:1784948815.285861:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784948815.285869:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000001:0.0:1784948815.285875:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784948815.285877:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558290432 : -115544151261184 : ffff96e9c7e88000) 00000400:00000001:0.0:1784948815.285884:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1784948815.285885:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f7830c1e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289654400:12345-0@lo:400: Request processed in 533us (1139us total) trans 0 rc 0/0 00000400:00000010:0.0:1784948815.285891:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000068d5a5a2. 00000100:00000040:2.0:1784948815.285894:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001c81e7dd x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:476/0 lens 224/0 e 0 to 0 dl 1784948826 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948815.285946:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000100:00100000:3.0:1784948815.285948:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 710 00000400:00000010:0.0:1784948815.285955:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000040:3.0:1784948815.285961:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948815.285968:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784948815.285971:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948815.285979:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784948815.285981:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:0.0:1784948815.285982:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948815.285987:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003f7291f9. 00000020:00000010:2.0:1784948815.285990:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000020:00000010:3.0:1784948815.286000:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000100:00000001:2.0:1784948815.286001:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948815.286011:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00000200:0.0:1784948815.286012:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948815.286013:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eb3e5fa2. 00000020:00000040:3.0:1784948815.286024:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00100000:2.0:1784948815.286027:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001c81e7dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289654528:12345-192.168.202.137@tcp:400: 02000000:00000001:3.0:1784948815.286033:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948815.286040:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 00000100:00000200:2.0:1784948815.286046:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289654528 02000000:00000001:3.0:1784948815.286050:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:2.0:1784948815.286054:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784948815.286057:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948815.286063:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:3.0:1784948815.286070:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948815.286071:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1784948815.286071:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948815.286078:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948815.286081:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:3.0:1784948815.286083:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b69fa69e. 00000100:00000001:0.0:1784948815.286087:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:2.0:1784948815.286088:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000010:3.0:1784948815.286093:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e6a263f4. 02000000:00000001:0.0:1784948815.286096:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:2.0:1784948815.286097:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784948815.286105:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:2.0:1784948815.286106:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948815.286111:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948815.286113:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948815.286119:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:0.0:1784948815.286120:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948815.286124:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948815.286129:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=137 v=5 (5 5 5 5) 00000020:00000001:2.0:1784948815.286130:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948815.286137:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00001000:0.0:1784948815.286139:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=137 v=5 (1 1 1 1) 02000000:00000001:2.0:1784948815.286141:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784948815.286148:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:2.0:1784948815.286150:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000ebcf220. 00000100:00000001:0.0:1784948815.286152:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948815.286157:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948815.286160:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:2.0:1784948815.286163:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948815.286165:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948815.286168:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948815.286171:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948815.286173:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289654528 00010000:00000001:2.0:1784948815.286180:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:0.0:1784948815.286184:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:2.0:1784948815.286190:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001c81e7dd x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:476/0 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948815.286238:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948815.286241:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784948815.286242:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948815.286251:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00001000:2.0:1784948815.286253:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=132 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948815.286258:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948815.286262:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948815.286269:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948815.286270:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948815.286278:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948815.286286:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948815.286290:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.286298:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784948815.286308:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000c8b06f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289654400:0@lo:400: 00000100:00000040:2.0:1784948815.286310:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948815.286317:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000001:0.0:1784948815.286324:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948815.286325:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289654528, offset 224 00000100:00000001:0.0:1784948815.286331:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948815.286337:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784948815.286339:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000c8b06f4 x1871651289654400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948815.286355:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784948815.286356:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948815.286363:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:2.0:1784948815.286370:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3275:3275:241:4294967295] 192.168.202.137@tcp LPNI seq info [2119:2119:8:4294967295] 02000000:00000010:0.0:1784948815.286371:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000f1904ca. 02000000:00000001:0.0:1784948815.286382:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948815.286386:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948815.286392:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948815.286401:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:2.0:1784948815.286402:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000010:0.0:1784948815.286407:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000008746b1e6. 00000800:00000200:2.0:1784948815.286415:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784948815.286417:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948815.286422:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000010:2.0:1784948815.286423:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c03b4f3a. 00000100:00000010:0.0:1784948815.286424:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000c8b06f4. 00000100:00000001:0.0:1784948815.286433:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000800:00000200:2.0:1784948815.286437:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948815.286439:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948815.286448:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948815.286454:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:0.0:1784948815.286458:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:2.0:1784948815.286460:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c03b4f3a type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948815.286466:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948815.286473:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948815.286477:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948815.286509:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948815.286515:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948815.286519:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948815.286521:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948815.286524:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948815.286529:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001c81e7dd x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:476/0 lens 224/224 e 0 to 0 dl 1784948826 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948815.286541:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001c81e7dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289654528:12345-192.168.202.137@tcp:400: Request processed in 560us (1064us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948815.286551:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 686 00000100:00000040:2.0:1784948815.286554:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948815.286557:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948815.286560:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948815.286564:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000ced696a. 00000020:00000010:2.0:1784948815.286570:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948815.286574:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000020:00000040:2.0:1784948815.286578:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948815.286582:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948815.286696:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948815.286704:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c03b4f3a. 00000400:00000200:3.0:1784948815.286711:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948815.286719:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948815.286721:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948815.286730:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:3.0:1784948815.286734:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000800:00000001:2.0:1784948815.286735:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948815.286743:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784948815.286744:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948815.286751:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948815.286753:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948815.286766:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948815.286779:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436500 00000400:00000200:2.0:1784948815.286788:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8431 [1] + 224 00000800:00000001:2.0:1784948815.286795:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948815.286809:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948815.286812:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948815.286816:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948815.286821:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948815.286825:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948815.286837:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948815.286859:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948815.286865:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948815.286868:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948815.286949:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948815.286955:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948815.286962:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948815.286965:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948815.286967:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006be72eea. 00000400:00000200:1.0:1784948815.286974:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784948815.286977:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784948815.286981:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948815.286988:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948815.287002:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948815.287012:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948815.287014:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948815.287017:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948815.287019:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948815.287021:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948815.287023:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948815.287026:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948815.287032:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=132 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948815.287036:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=132 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948815.287039:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948815.287040:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948815.287043:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948815.287049:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948815.287063:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948815.287069:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948815.287073:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948815.287076:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948815.287079:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948815.287084:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948815.287098:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948815.287103:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948815.287105:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948815.287111:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948815.287126:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289654528:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948815.287134:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948815.287138:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948815.287142:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f59e6b47 x1871651289654528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948815.287156:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948815.287159:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948815.287163:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dbce179e. 02000000:00000001:1.0:1784948815.287173:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948815.287175:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948815.287178:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948815.287182:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948815.287184:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002aeee8e9. 02000000:00000001:1.0:1784948815.287188:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948815.287190:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948815.287191:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f59e6b47. 00000100:00000001:1.0:1784948815.287195:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948815.287196:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948815.287199:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948815.287205:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948815.287207:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948815.287209:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948815.287212:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948815.328032:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305754667, at 4305754668 00000100:00000040:1.0:1784948815.328032:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305754667, at 4305754668 00000100:00000040:3.0:1784948815.328039:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305754667, at 4305754668 00000100:00000040:2.0:1784948815.328042:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305754667, at 4305754668 00000100:00000010:1.0:1784948815.328057:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000010:0.0:1784948815.328062:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:3.0:1784948815.328070:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000004f2eb99f. 00000100:00000010:2.0:1784948815.328074:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005975f076. 00010000:00000001:2.0:1784948816.116025:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948816.116035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948816.116044:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948816.116046:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948816.116174:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948816.116181:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948816.116185:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948816.308058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948816.308076:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948816.308080:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948816.308089:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948816.308092:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948816.308096:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948816.308099:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1784948816.820052:0:343957:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.25:988 00000400:00000100:2.0:1784948816.820135:0:343957:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.25@tcp at host 111.222.173.25:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948816.820150:0:343957:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.25@tcp: down 00000800:00000200:2.0:1784948816.820163:0:343957:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.173.25@tcp down 00010000:00000001:2.0:1784948817.140047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948817.140056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948817.140064:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948817.140067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948817.140136:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948817.140143:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948817.140147:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948817.332037:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948817.332054:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948817.332057:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948817.332062:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948817.332066:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948817.332070:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948817.332087:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948818.164006:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948818.164014:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948818.164021:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948818.164022:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948818.164096:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948818.164102:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948818.164106:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948818.356014:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948818.356029:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948818.356032:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948818.356038:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948818.356042:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948818.356047:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948818.356049:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948819.188077:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948819.188090:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948819.188101:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948819.188103:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948819.188203:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948819.188213:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948819.188219:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948819.380036:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948819.380047:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784948819.380067:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948794/real 1784948794] req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948819 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784948819.380101:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948819.380105:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948819.380110:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ecb3ce7f. 00000400:00000200:2.0:1784948819.380122:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000236111b7 00000400:00000010:2.0:1784948819.380126:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000236111b7. 00000100:00000001:2.0:1784948819.380134:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948819.380141:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948819 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948819.380156:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948819 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948819.380175:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948819.380179:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.380183:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948819.380186:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948819.380190:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948819 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948819.380200:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.380212:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948819.380215:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948819.380222:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948819.380227:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948819.380231:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948819.380240:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948819 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948819.380255:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784948819.380262:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.380265:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948819.380268:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.380273:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784948819.380278:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784948819.380284:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784948819.380322:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:2.0:1784948819.380328:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784948819.380337:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948819 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948819.380349:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289653376:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784948819.380356:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.380361:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948819.380366:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e8f14423 x1871651289653376/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948819 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948819.380377:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948819.380381:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948819.380386:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000b9f7c212. 02000000:00000001:2.0:1784948819.380394:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948819.380397:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948819.380401:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784948819.380405:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948819.380409:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000fed721f2. 02000000:00000001:2.0:1784948819.380415:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948819.380417:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948819.380419:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e8f14423. 00000100:00000001:2.0:1784948819.380425:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948819.380427:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.380430:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948819.380436:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948819.380437:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948819.380443:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.380479:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948819.380482:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948819.380487:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948819.380492:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e8f14423. 00000020:00000040:1.0:1784948819.380501:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948819.380508:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948819.380513:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948819.380519:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948819.380530:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e79c6895. 00000100:00000001:1.0:1784948819.380539:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948819.380547:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948819.380561:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948819.380564:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948819.380571:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e8f14423] to pc [ptlrpcd_00_03+3] req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948819.380623:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948819.380628:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948819.380632:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948819.380636:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948819.380640:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948819.380645:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784948819.380651:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11066 00000100:00000001:0.0:1784948819.380660:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948819.380665:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:3.0:1784948819.380665:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948819.380670:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948819.380675:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.380678:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1784948819.380680:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11065 00000100:00000040:0.0:1784948819.380693:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948819.380708:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:0.0:1784948819.380715:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784948819.380719:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:0.0:1784948819.380721:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948819.380729:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:0.0:1784948819.380731:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948819.380734:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784948819.380737:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948819.380744:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:0.0:1784948819.380746:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.380753:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00100000:0.0:1784948819.380758:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289654656:0@lo:400: 00000020:00000001:1.0:1784948819.380761:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948819.380766:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:0.0:1784948819.380771:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1784948819.380773:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948819.380780:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:0.0:1784948819.380781:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1784948819.380785:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:0.0:1784948819.380787:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948819.380792:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:0.0:1784948819.380794:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784948819.380805:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000010:0.0:1784948819.380806:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eff166f6. 00000100:00000001:1.0:1784948819.380808:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784948819.380817:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948819.380819:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948819.380825:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:0.0:1784948819.380828:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d163afc3. 00000020:00000040:1.0:1784948819.380829:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:1.0:1784948819.380841:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000010:0.0:1784948819.380844:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000100:00080000:1.0:1784948819.380847:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:1.0:1784948819.380855:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948819.380857:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289654656, portal 25 02000000:00000001:1.0:1784948819.380859:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1784948819.380866:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948819.380869:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451655040 : -115544257896576 : ffff96e9c18d5f80) 02000000:00000001:1.0:1784948819.380870:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948819.380877:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948819.380881:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948819.380883:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1784948819.380889:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948819.380902:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b93d109. 00000100:00000001:0.0:1784948819.380943:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948819.380951:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948819.380957:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000020:00000040:1.0:1784948819.380961:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:0.0:1784948819.380964:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289654656, offset 0 02000000:00000001:1.0:1784948819.380975:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1784948819.380983:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784948819.380984:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.380993:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948819.380997:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948819.381002:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948819.381005:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:1.0:1784948819.381018:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000afc15cec. 00000400:00000200:0.0:1784948819.381028:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436580 00000100:00000001:1.0:1784948819.381030:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948819.381043:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7a25 [2] + 0 00000400:00000010:0.0:1784948819.381053:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000100:00100000:1.0:1784948819.381053:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784948819.381067:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000001ee8a23e 00000400:00000200:0.0:1784948819.381082:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948819.381084:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948819.381093:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948819.381102:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:1.0:1784948819.381112:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005b93d109] to pc [ptlrpcd_rcv+-1] req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948819.381113:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001ee8a23e 00000400:00000010:0.0:1784948819.381121:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000001ee8a23e. 00000100:00000001:0.0:1784948819.381134:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948819.381140:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948819.381148:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000002b339182 x1871651289654656 msgsize 224 00000100:00100000:0.0:1784948819.381159:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:1.0:1784948819.381163:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948819.381164:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784948819.381175:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948819.381176:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948819.381187:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948819.381188:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948819.381189:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948819.381196:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1784948819.381203:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000457a2a3. 00000400:00000200:0.0:1784948819.381204:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000100:00000040:2.0:1784948819.381208:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948819.381213:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000020:00000040:1.0:1784948819.381219:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948819.381223:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948819.381234:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1784948819.381236:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948819.381244:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948819.381246:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1784948819.381253:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948819.381254:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381268:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1784948819.381269:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:1.0:1784948819.381272:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000adf08f9b. 02000000:00000001:2.0:1784948819.381276:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948819.381278:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948819.381284:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948819.381287:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948819.381305:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289654784:1.2.3.4@tcp:8: 00000100:00000040:1.0:1784948819.381306:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948819.381307:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381318:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784948819.381324:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784948819.381333:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381334:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.381337:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948819.381343:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381344:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305758763 00000100:00000001:2.0:1784948819.381345:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948819.381358:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948819.381359:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000457a2a3] to pc [ptlrpcd_00_00+0] req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784948819.381360:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:2.0:1784948819.381363:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948819.381364:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948819.381375:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289654656 00000100:00000001:0.0:1784948819.381379:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784948819.381379:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784948819.381383:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948819.381387:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948819.381391:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948819.381397:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305758763 00000100:00000001:3.0:1784948819.381399:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948819.381400:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784948819.381406:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:2.0:1784948819.381408:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784948819.381411:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.381417:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381418:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1784948819.381420:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289654656 00000020:00000001:3.0:1784948819.381429:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:1.0:1784948819.381430:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11096) 00000100:00000001:0.0:1784948819.381433:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305758763 00000020:00000040:3.0:1784948819.381438:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:0.0:1784948819.381445:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000020:00000001:3.0:1784948819.381446:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948819.381478:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948819.381481:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000001:3.0:1784948819.381487:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784948819.381494:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948819.381496:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305758763 00000400:00000200:1.0:1784948819.381504:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:3.0:1784948819.381504:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:0.0:1784948819.381506:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000020:00000010:3.0:1784948819.381515:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001f45750. 00000100:00000001:0.0:1784948819.381517:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000200:1.0:1784948819.381519:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000001:3.0:1784948819.381531:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948819.381534:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948819.381536:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009f047fe3. 00000020:00000010:3.0:1784948819.381538:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001f45750. 00000100:00000001:0.0:1784948819.381542:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948819.381550:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00000001:0.0:1784948819.381552:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948819.381556:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eb3e5fa2. 00000100:00000001:0.0:1784948819.381562:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948819.381571:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000cf871ac3. 00000020:00000010:3.0:1784948819.381572:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000565c53c7. 00000100:00000001:0.0:1784948819.381573:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948819.381579:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948819.381587:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:3.0:1784948819.381588:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000400:00000200:1.0:1784948819.381589:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784948819.381592:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948819.381603:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381610:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948819.381614:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:1.0:1784948819.381618:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:3.0:1784948819.381622:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948819.381627:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:1.0:1784948819.381629:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784948819.381631:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948819.381632:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948819.381641:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000400:00000200:1.0:1784948819.381641:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:2.0:1784948819.381649:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948819.381649:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381651:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948819.381656:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784948819.381661:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.381666:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784948819.381668:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:0.0:1784948819.381673:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948819.381675:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948819.381683:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948819.381684:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3276:3276:241:4294967295] 1.2.3.4@tcp LPNI seq info [99:99:8:4294967295] 00000100:00000001:0.0:1784948819.381689:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948819.381697:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948819.381698:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948819.381700:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:1.0:1784948819.381703:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000001:2.0:1784948819.381707:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948819.381709:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784948819.381716:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948819.381727:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 711 00000400:00000200:1.0:1784948819.381739:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000040:3.0:1784948819.381741:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948819.381753:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782154832 : -115538927396784 : ffff96eaff465850) 00000100:00000001:0.0:1784948819.381754:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948819.381760:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948819.381765:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 02000000:00000001:0.0:1784948819.381769:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1784948819.381771:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000002b339182 x1871651289654656/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:480/0 lens 224/0 e 0 to 0 dl 1784948830 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948819.381777:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1784948819.381778:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 02000000:00000001:0.0:1784948819.381785:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948819.381796:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00100000:0.0:1784948819.381797:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000457a2a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289654912:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948819.381819:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000800:00000200:1.1:1784948819.381821:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 02000000:00000001:0.0:1784948819.381825:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948819.381832:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948819.381837:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948819.381838:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000010:0.0:1784948819.381844:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 00000020:00000010:3.0:1784948819.381846:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000001f45750. 02000000:00000001:0.0:1784948819.381852:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948819.381855:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948819.381861:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000400:00000010:0.0:1784948819.381862:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b85c9890. 00000020:00000010:3.0:1784948819.381864:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000001f45750. 00000400:00000200:1.0:1784948819.381874:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 00000400:00000010:0.0:1784948819.381875:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000100:00100000:3.0:1784948819.381885:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000002b339182 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289654656:12345-0@lo:400: 00000100:00000200:0.0:1784948819.381888:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289654912, portal 10 00000100:00000001:0.0:1784948819.381898:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000200:3.0:1784948819.381900:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289654656 00000100:00000001:0.0:1784948819.381946:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529525959168 : -115544183592448 : ffff96e9c5fb2a00) 00000020:00000001:3.0:1784948819.381950:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948819.381958:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:0.0:1784948819.381961:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948819.381964:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948819.381975:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948819.381982:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784948819.381983:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1784948819.381991:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000040:0.0:1784948819.381996:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784948819.381998:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948819.382004:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000020:00000001:3.0:1784948819.382005:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948819.382011:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:0.0:1784948819.382013:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289654912, offset 0 00000020:00000001:3.0:1784948819.382016:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948819.382022:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:0.0:1784948819.382027:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948819.382029:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948819.382035:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:0.0:1784948819.382040:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000010:3.0:1784948819.382043:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d8bbc1a8. 00000400:00000200:0.0:1784948819.382049:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1784948819.382051:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948819.382057:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948819.382058:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:3.0:1784948819.382062:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948819.382068:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00010000:00000040:3.0:1784948819.382070:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289654656 00000400:00000200:0.0:1784948819.382077:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00010000:00000001:3.0:1784948819.382079:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:0.0:1784948819.382087:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3277:3277:240:4294967295] 192.168.202.137@tcp LPNI seq info [2120:2120:8:4294967295] 00010000:00000200:3.0:1784948819.382089:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000002b339182 x1871651289654656/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:480/0 lens 224/224 e 0 to 0 dl 1784948830 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948819.382109:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948819.382115:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948819.382116:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00001000:3.0:1784948819.382124:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=141 v=5 (1 1 1 1) 00000800:00000200:0.0:1784948819.382131:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948819.382135:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:0.0:1784948819.382140:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008746b1e6. 00000100:00000040:3.0:1784948819.382142:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948819.382150:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000200:0.0:1784948819.382158:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:3.0:1784948819.382159:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948819.382166:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948819.382172:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000200:0.0:1784948819.382177:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:3.0:1784948819.382186:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000200:0.0:1784948819.382188:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008746b1e6 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:3.0:1784948819.382194:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000200:3.0:1784948819.382203:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289654656, offset 224 00000100:00000001:0.0:1784948819.382208:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948819.382216:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:0.0:1784948819.382220:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2055 usecs (3 jiffies) 00000100:00000001:0.0:1784948819.382227:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305758725 00000100:00000001:0.0:1784948819.382232:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305758763 00000400:00000200:3.0:1784948819.382236:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948819.382240:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305758725 00000100:00000001:0.0:1784948819.382246:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305758763 00000100:00000001:0.0:1784948819.382253:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305758725 00000400:00000200:3.0:1784948819.382254:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436580 00000100:00000001:0.0:1784948819.382261:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305758763 00000100:00000001:0.0:1784948819.382268:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305758725 00000400:00000200:3.0:1784948819.382273:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8445 [1] + 224 00000100:00000001:0.0:1784948819.382276:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305758763 00000100:00000001:0.0:1784948819.382285:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948819.382290:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948819.382291:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948819.382296:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948819.382303:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948819.382307:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948819.382311:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948819.382320:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948819.382333:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948819.382350:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1784948819.382359:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948819.382362:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000800:00000200:2.0:1784948819.382369:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948819.382369:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948819.382376:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000010:2.0:1784948819.382378:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008746b1e6. 00000100:00000001:3.0:1784948819.382386:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948819.382389:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948819.382390:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948819.382395:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:3.0:1784948819.382401:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:2.0:1784948819.382407:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784948819.382408:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948819.382410:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948819.382419:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:2.0:1784948819.382420:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000800:00000001:0.0:1784948819.382421:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948819.382429:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002b339182 x1871651289654656/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:480/0 lens 224/224 e 0 to 0 dl 1784948830 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948819.382430:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000800:00000001:0.0:1784948819.382432:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.382442:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784948819.382482:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784948819.382483:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948819.382493:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000002b339182 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289654656:12345-0@lo:400: Request processed in 647us (1335us total) trans 0 rc 0/0 00000400:00000200:0.0:1784948819.382513:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436680 00000100:00100000:3.0:1784948819.382527:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 711 00000400:00000200:0.0:1784948819.382536:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 154928 00000100:00000001:2.0:1784948819.382542:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948819.382542:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000800:00000001:0.0:1784948819.382546:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948819.382551:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000040:2.0:1784948819.382554:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948819.382557:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:0.0:1784948819.382560:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948819.382566:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000565c53c7. 00000400:00000200:0.0:1784948819.382571:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784948819.382580:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000100:00000001:2.0:1784948819.382588:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948819.382590:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784948819.382591:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eb3e5fa2. 00000100:00000001:2.0:1784948819.382594:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:3.0:1784948819.382605:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:3.0:1784948819.382615:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784948819.382618:0:337114:0:(events.c:286:request_in_callback()) Process entered 02000000:00000010:3.0:1784948819.382619:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d8bbc1a8. 00000100:00000200:0.0:1784948819.382624:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000001:3.0:1784948819.382628:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000010:0.0:1784948819.382632:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000c8b06f4. 00000100:00000001:3.0:1784948819.382633:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948819.382641:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000000c8b06f4 x1871651289654912 msgsize 224 00000100:00000200:3.0:1784948819.382644:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:0.0:1784948819.382656:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000010:3.0:1784948819.382657:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fed0cd52. 00000400:00000010:3.0:1784948819.382666:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000047756a57. 00000100:00000001:0.0:1784948819.382674:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784948819.382687:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948819.382694:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948819.382717:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948819.382720:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948819.382724:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948819.382727:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948819.382730:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d163afc3. 00000400:00000200:0.0:1784948819.382737:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:0.0:1784948819.382740:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00000001:1.0:1784948819.382742:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784948819.382744:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784948819.382750:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289654912 00000100:00000200:0.0:1784948819.382756:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948819.382759:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948819.382765:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948819.382770:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948819.382781:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948819.382791:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289654912 00000020:00000001:1.0:1784948819.382798:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948819.382804:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000200:0.0:1784948819.382810:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948819.382812:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948819.382819:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948819.382828:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948819.382836:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948819.382841:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948819.382843:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948819.382846:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948819.382853:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:0.0:1784948819.382854:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948819.382860:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948819.382861:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948819.382865:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:1.0:1784948819.382867:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:0.0:1784948819.382870:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948819.382876:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948819.382877:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 00000100:00001000:0.0:1784948819.382884:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=141 v=5 (5 5 5 5) 00000020:00000010:1.0:1784948819.382887:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 00000100:00001000:0.0:1784948819.382894:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=141 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948819.382900:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948819.382901:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948819.382936:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948819.382939:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.382943:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948819.382947:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948819.382956:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948819.382959:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948819.382960:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948819.382964:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948819.382972:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289654656:0@lo:400: 00000100:00000001:0.0:1784948819.382979:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948819.382981:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948819.382983:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e8f14423 x1871651289654656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948819.382991:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948819.382994:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948819.382997:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eff166f6. 02000000:00000001:0.0:1784948819.383002:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948819.383006:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948819.383008:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948819.383011:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948819.383013:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e79c6895. 02000000:00000001:0.0:1784948819.383017:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948819.383019:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948819.383020:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e8f14423. 00000100:00000001:0.0:1784948819.383025:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948819.383028:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948819.383030:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948819.383035:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948819.383037:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948819.383040:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948819.383043:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948819.383055:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:1.0:1784948819.383075:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948819.383080:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948819.383082:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948819.383087:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948819.383093:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948819.383096:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948819.383098:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.383114:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948819.383126:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948819.383128:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948819.383134:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 687 00000100:00000040:1.0:1784948819.383137:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948819.383140:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558297600 : -115544151254016 : ffff96e9c7e89c00) 00000100:00000040:1.0:1784948819.383147:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000c8b06f4 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:480/0 lens 224/0 e 0 to 0 dl 1784948830 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948819.383159:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948819.383164:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:1.0:1784948819.383167:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948819.383171:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00100000:1.0:1784948819.383178:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000c8b06f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289654912:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948819.383183:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289654912 00000020:00000001:1.0:1784948819.383186:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948819.383190:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948819.383192:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948819.383194:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948819.383197:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948819.383200:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948819.383203:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948819.383206:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948819.383207:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948819.383210:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948819.383212:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948819.383217:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948819.383219:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948819.383223:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000825c5cf0. 02000000:00000001:1.0:1784948819.383226:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.383228:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948819.383230:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948819.383233:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289654912 00010000:00000001:1.0:1784948819.383236:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948819.383240:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000c8b06f4 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:480/0 lens 224/224 e 0 to 0 dl 1784948830 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948819.383249:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948819.383252:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948819.383254:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=136 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948819.383259:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948819.383261:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948819.383264:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948819.383267:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948819.383269:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948819.383271:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948819.383274:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948819.383278:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000200:1.0:1784948819.383282:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289654912, offset 224 00000400:00000200:1.0:1784948819.383287:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948819.383296:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948819.383302:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3278:3278:240:4294967295] 192.168.202.137@tcp LPNI seq info [2121:2121:8:4294967295] 00000400:00000200:1.0:1784948819.383319:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948819.383328:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948819.383335:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000800:00000200:1.0:1784948819.383343:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948819.383352:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948819.383358:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948819.383382:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948819.383389:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948819.383393:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948819.383396:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948819.383400:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948819.383406:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c8b06f4 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:480/0 lens 224/224 e 0 to 0 dl 1784948830 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948819.383424:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000c8b06f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289654912:12345-192.168.202.137@tcp:400: Request processed in 262us (780us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948819.383438:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 687 00000100:00000040:1.0:1784948819.383443:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948819.383447:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948819.383476:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948819.383480:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000020:00000010:1.0:1784948819.383487:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948819.383492:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00000020:00000040:1.0:1784948819.383498:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948819.383502:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948819.383651:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948819.383660:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000400:00000200:2.0:1784948819.383670:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948819.383682:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948819.383688:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948819.383691:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:2.0:1784948819.383699:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000800:00000001:0.0:1784948819.383705:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.383707:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784948819.383712:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948819.383714:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948819.383719:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948819.383733:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948819.383749:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436680 00000400:00000200:0.0:1784948819.383758:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8451 [1] + 224 00000800:00000001:0.0:1784948819.383765:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948819.383775:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948819.383777:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948819.383782:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948819.383787:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948819.383793:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948819.383806:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948819.383836:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948819.383843:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948819.383846:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.383902:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948819.383934:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948819.383939:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948819.383942:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948819.383945:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b85c9890. 00000400:00000200:2.0:1784948819.383950:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000400:00000010:2.0:1784948819.383952:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000100:00000001:2.0:1784948819.383957:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948819.383963:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948819.383976:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948819.383986:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948819.383988:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.383990:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948819.383993:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948819.383995:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948819.383997:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948819.384000:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948819.384006:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=136 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948819.384010:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=136 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948819.384013:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948819.384015:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948819.384017:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948819.384022:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948819.384031:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948819.384034:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948819.384037:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948819.384039:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948819.384041:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948819.384044:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948819.384052:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948819.384055:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948819.384056:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948819.384059:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948819.384068:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000457a2a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289654912:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948819.384074:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.384076:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948819.384078:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000457a2a3 x1871651289654912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948835 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948819.384086:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948819.384088:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948819.384090:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 02000000:00000001:2.0:1784948819.384094:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948819.384095:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948819.384097:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948819.384100:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948819.384102:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000adf08f9b. 02000000:00000001:2.0:1784948819.384105:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948819.384107:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948819.384108:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000457a2a3. 00000100:00000001:2.0:1784948819.384112:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948819.384113:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.384115:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948819.384120:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948819.384121:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948819.384124:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948819.384126:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948819.424021:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305758763, at 4305758764 00000100:00000040:2.0:1784948819.424024:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305758763, at 4305758764 00000100:00000040:0.0:1784948819.424038:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305758763, at 4305758764 00000100:00000010:3.0:1784948819.424039:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000010:2.0:1784948819.424043:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000010:0.0:1784948819.424058:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:1.0:1784948819.424061:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305758763, at 4305758764 00000100:00000010:1.0:1784948819.424081:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00010000:00000001:2.0:1784948820.212017:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948820.212027:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948820.212035:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948820.212039:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948820.212042:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948820.212046:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948820.212062:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948820.212065:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948820.212067:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948820.212070:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948820.212159:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948820.212169:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948820.212175:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948820.404255:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948820.404265:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948820.404280:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948819/real 0] req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948820.404299:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.404303:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948820.404306:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948820.404310:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948820.404319:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.404329:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948820.404331:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.404334:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948820.404337:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948820.404340:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948820.404344:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948820 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948820.404353:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784948820.404357:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.404359:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948820.404360:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.404363:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784948820.404366:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948820.404370:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784948820.404399:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784948820.404403:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784948820.404407:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948820 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948820.404417:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289654784:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784948820.404440:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.404475:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948820.404479:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005b93d109 x1871651289654784/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948820 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948820.404489:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784948820.404493:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948820.404496:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784948820.404500:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948820.404505:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000afc15cec. 02000000:00000001:0.0:1784948820.404513:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948820.404516:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948820.404518:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b93d109. 00000100:00000040:3.0:1784948820.404522:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948820.404524:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948820.404528:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.404533:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.404542:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948820.404544:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948820.404546:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.404553:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948820.404554:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948820.404559:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000055b3248. 00000020:00000040:3.0:1784948820.404569:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948820.404577:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948820.404583:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948820.404590:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948820.404601:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:3.0:1784948820.404612:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948820.404623:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948820.404643:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948820.404647:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948820.404655:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000055b3248] to pc [ptlrpcd_00_01+1] req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948820.404703:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948820.404708:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948820.404714:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948820.404718:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784948820.404720:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784948820.404726:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784948820.404733:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11066 00000400:00000200:3.0:1784948820.404747:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:3.0:1784948820.404754:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11091 00000400:00000200:3.0:1784948820.404761:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:3.0:1784948820.404766:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1784948820.404772:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948820.404779:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948820.404785:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948820.404790:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:3.0:1784948820.404796:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948820.404800:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:3.0:1784948820.404804:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784948820.404809:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948820.404809:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1784948820.404811:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:1.0:1784948820.404819:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.404819:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784948820.404822:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:2.0:1784948820.404827:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1784948820.404834:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:2.0:1784948820.404843:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948820.404850:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948820.404860:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948820.404871:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948820.404883:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784948820.404884:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948820.404891:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948820.404894:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:2.0:1784948820.404900:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948820.404935:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948820.404940:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948820.404944:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000055b3248 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289655040:0@lo:400: 00000100:00000001:2.0:1784948820.404950:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948820.404958:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1784948820.404960:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 02000000:00000001:2.0:1784948820.404963:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948820.404971:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1784948820.404976:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 02000000:00000010:2.0:1784948820.404981:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 00000100:00000001:3.0:1784948820.404985:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948820.404986:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948820.404999:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1784948820.405001:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009129b91b. 02000000:00000001:3.0:1784948820.405011:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784948820.405012:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 02000000:00000001:3.0:1784948820.405023:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948820.405026:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289655040, portal 25 00000100:00000001:3.0:1784948820.405029:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1784948820.405032:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948820.405037:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482522496 : -115544227029120 : ffff96e9c3645f80) 00000100:00000001:3.0:1784948820.405038:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948820.405046:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948820.405052:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b16a92e. 00000100:00000001:2.0:1784948820.405064:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1784948820.405066:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:2.0:1784948820.405070:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948820.405079:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 02000000:00000001:3.0:1784948820.405082:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1784948820.405085:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289655040, offset 0 02000000:00000001:3.0:1784948820.405089:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948820.405097:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948820.405099:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948820.405107:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948820.405113:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948820.405119:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:3.0:1784948820.405130:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d8bbc1a8. 00000400:00000200:2.0:1784948820.405137:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436700 00000100:00000001:3.0:1784948820.405143:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948820.405151:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7a45 [2] + 0 00000400:00000010:2.0:1784948820.405159:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000080bb5714. 00000100:00100000:3.0:1784948820.405164:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784948820.405166:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000ac3f6cae 00000400:00000200:2.0:1784948820.405179:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948820.405192:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948820.405197:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1784948820.405201:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ac3f6cae 00000100:00000001:3.0:1784948820.405204:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948820.405206:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000ac3f6cae. 00000100:00000001:2.0:1784948820.405219:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784948820.405225:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b16a92e] to pc [ptlrpcd_rcv+-1] req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948820.405226:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784948820.405232:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000e164874b x1871651289655040 msgsize 224 00000100:00100000:2.0:1784948820.405248:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948820.405253:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784948820.405279:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948820.405287:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948820.405295:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000100:00000040:3.0:1784948820.405297:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1784948820.405303:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000100:00000001:3.0:1784948820.405310:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948820.405315:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1784948820.405321:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b13d185. 00000100:00000200:2.0:1784948820.405326:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784948820.405332:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948820.405344:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784948820.405346:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:3.0:1784948820.405351:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1784948820.405353:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289655040 00000100:00000001:2.0:1784948820.405354:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784948820.405360:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:3.0:1784948820.405360:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948820.405362:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948820.405367:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948820.405373:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948820.405374:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 02000000:00000001:1.0:1784948820.405382:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.405383:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948820.405388:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.405389:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948820.405390:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1784948820.405392:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289655040 00000100:00000001:0.0:1784948820.405397:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1784948820.405401:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948820.405403:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948820.405404:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948820.405410:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:3.0:1784948820.405410:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948820.405416:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305759787 00000100:00000040:0.0:1784948820.405419:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948820.405421:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948820.405429:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:2.0:1784948820.405429:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000020:00000001:1.0:1784948820.405439:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948820.405472:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948820.405480:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1784948820.405483:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948820.405486:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.405490:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305759787 02000000:00000001:0.0:1784948820.405497:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948820.405498:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:2.0:1784948820.405503:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000824e9543. 02000000:00000001:0.0:1784948820.405505:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948820.405509:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000010:1.0:1784948820.405511:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:2.0:1784948820.405511:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1784948820.405512:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948820.405517:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948820.405522:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948820.405525:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305759787 00000100:00100000:0.0:1784948820.405527:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289655168:192.168.202.137@tcp:8: 00000020:00000010:1.0:1784948820.405533:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00000010:2.0:1784948820.405533:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000abbd7d53. 00000100:00000040:3.0:1784948820.405535:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b13d185] to pc [ptlrpcd_00_02+2] req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948820.405540:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784948820.405545:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:0.0:1784948820.405547:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000010:1.0:1784948820.405549:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f7a7941e. 02000000:00000001:0.0:1784948820.405554:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.405557:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305759787 00000020:00000010:1.0:1784948820.405561:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 02000000:00000001:0.0:1784948820.405564:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1784948820.405569:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f68fd7be. 00000020:00000010:1.0:1784948820.405573:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 02000000:00000010:0.0:1784948820.405575:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000af0b2fc8. 00000100:00000001:2.0:1784948820.405581:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:0.0:1784948820.405586:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948820.405591:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948820.405592:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948820.405598:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d163afc3. 00000100:00000001:1.0:1784948820.405600:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948820.405600:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948820.405607:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948820.405609:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948820.405611:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000100:00000001:1.0:1784948820.405613:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.405613:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948820.405619:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289655168, portal 4 00000100:00000001:3.0:1784948820.405625:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.405626:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948820.405626:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948820.405628:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.405634:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482521600 : -115544227030016 : ffff96e9c3645c00) 00000100:00000001:2.0:1784948820.405635:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:3.0:1784948820.405643:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11097) 00000100:00000001:2.0:1784948820.405647:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948820.405652:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948820.405655:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948820.405656:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948820.405662:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.405674:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948820.405674:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948820.405677:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948820.405680:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948820.405683:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948820.405686:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784948820.405689:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948820.405693:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:0.0:1784948820.405696:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00000200:0.0:1784948820.405705:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289655168, offset 0 00000100:00100000:1.0:1784948820.405705:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 712 00000100:00000040:3.0:1784948820.405709:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948820.405721:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948820.405727:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784948820.405728:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948820.405731:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782158928 : -115538927392688 : ffff96eaff466850) 00000100:00000001:2.0:1784948820.405735:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948820.405750:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e164874b x1871651289655040/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:481/0 lens 224/0 e 0 to 0 dl 1784948831 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948820.405751:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784948820.405755:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948820.405763:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948820.405767:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1784948820.405779:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948820.405784:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:3.0:1784948820.405787:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948820.405793:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:1.0:1784948820.405794:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 02000000:00000001:3.0:1784948820.405799:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948820.405804:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784948820.405805:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784948820.405815:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b13d185 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289655296:192.168.202.137@tcp:400: 00000400:00000200:0.0:1784948820.405817:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:1.0:1784948820.405818:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000400:00000200:0.0:1784948820.405833:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3279:3279:240:4294967295] 192.168.202.137@tcp LPNI seq info [2122:2122:8:4294967295] 00000100:00000001:3.0:1784948820.405835:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1784948820.405840:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e164874b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289655040:12345-0@lo:400: 02000000:00000001:3.0:1784948820.405856:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000200:1.0:1784948820.405861:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289655040 02000000:00000001:3.0:1784948820.405863:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948820.405887:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:3.0:1784948820.405888:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948820.405892:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:1.0:1784948820.405896:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000010:3.0:1784948820.405900:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 00000020:00000001:1.0:1784948820.405901:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948820.405906:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784948820.405911:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:3.0:1784948820.405913:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948820.405916:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000010:0.0:1784948820.405917:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008746b1e6. 00000020:00000001:1.0:1784948820.405959:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948820.405963:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948820.405964:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4db7024. 00000800:00000200:0.0:1784948820.405970:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:1.0:1784948820.405973:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948820.405978:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:3.0:1784948820.405980:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000020:00000001:1.0:1784948820.405982:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948820.405991:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:1.0:1784948820.405992:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000200:3.0:1784948820.405995:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289655296, portal 10 00000100:00000001:1.0:1784948820.406000:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000200:0.0:1784948820.406001:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008746b1e6 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1784948820.406006:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1784948820.406007:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784948820.406014:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482526080 : -115544227025536 : ffff96e9c3646d80) 02000000:00000010:1.0:1784948820.406018:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000afc15cec. 00000100:00000001:0.0:1784948820.406025:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948820.406026:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948820.406034:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948820.406039:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:1.0:1784948820.406039:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.406051:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305759788 00000020:00000001:1.0:1784948820.406053:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.406064:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305759787 00010000:00000040:1.0:1784948820.406066:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289655040 00000100:00000001:3.0:1784948820.406073:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948820.406077:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305759788 00010000:00000001:1.0:1784948820.406077:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:3.0:1784948820.406088:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948820.406090:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305759787 00010000:00000200:1.0:1784948820.406092:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e164874b x1871651289655040/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:481/0 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948820.406099:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305759788 00000400:00000010:3.0:1784948820.406100:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 00000100:00000001:0.0:1784948820.406109:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305759787 00000100:00000200:3.0:1784948820.406111:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289655296, offset 0 00000100:00000001:0.0:1784948820.406121:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305759788 00000100:00000001:0.0:1784948820.406131:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305759787 00010000:00000001:1.0:1784948820.406135:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948820.406139:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948820.406143:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00010000:00000001:1.0:1784948820.406144:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948820.406145:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948820.406158:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=142 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948820.406159:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948820.406164:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784948820.406166:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948820.406177:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948820.406179:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1784948820.406181:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784948820.406185:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000040:1.0:1784948820.406187:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948820.406192:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784948820.406196:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784948820.406198:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948820.406200:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:0.0:1784948820.406204:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948820.406208:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1784948820.406211:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784948820.406212:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948820.406220:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948820.406220:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1784948820.406221:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000800:00000010:2.0:1784948820.406230:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008746b1e6. 00000100:00000001:1.0:1784948820.406231:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784948820.406238:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3280:3280:239:4294967295] 192.168.202.137@tcp LPNI seq info [2123:2123:7:4294967295] 00000100:00000040:1.0:1784948820.406240:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:2.0:1784948820.406241:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1784948820.406249:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000800:00000001:0.0:1784948820.406255:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948820.406257:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:1.0:1784948820.406258:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289655040, offset 224 00000800:00000001:0.0:1784948820.406272:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948820.406275:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:0.0:1784948820.406281:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948820.406282:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000001:0.0:1784948820.406290:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948820.406298:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000800:00000200:3.0:1784948820.406303:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948820.406306:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000400:00000200:0.0:1784948820.406309:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948820.406311:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784948820.406317:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000010:3.0:1784948820.406320:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000200:2.0:1784948820.406330:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948820.406344:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436700 00000400:00000200:0.0:1784948820.406345:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6436780 00000800:00000200:3.0:1784948820.406346:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784948820.406360:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8465 [1] + 224 00000100:00000001:2.0:1784948820.406380:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784948820.406388:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:2.0:1784948820.406390:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948845 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948820.406393:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1784948820.406405:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000400:00000100:0.0:1784948820.406406:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289655168 offset 0 length 520: 4 00000400:00000200:1.0:1784948820.406407:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948820.406425:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000009456042d is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:1.0:1784948820.406425:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784948820.406429:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948820.406436:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948820.406437:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:2.0:1784948820.406437:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948820.406484:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948820.406492:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948820.406499:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2057 usecs (3 jiffies) 00000800:00000001:0.0:1784948820.406503:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948820.406513:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948820.406513:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305759749 00000100:00000040:1.0:1784948820.406516:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948820.406525:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305759788 00000100:00000001:3.0:1784948820.406536:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305759749 00000100:00000001:3.0:1784948820.406544:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305759788 00000100:00000001:3.0:1784948820.406553:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305759749 00000100:00000001:3.0:1784948820.406557:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305759788 00000100:00000001:3.0:1784948820.406560:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305759749 00000100:00000001:3.0:1784948820.406562:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305759788 00000100:00000001:3.0:1784948820.406566:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948820.406570:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948820.406575:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948820.406579:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948820.406580:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.406594:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784948820.406595:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000100:00000001:3.0:1784948820.406599:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000010:1.0:1784948820.406602:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000800:00000200:2.0:1784948820.406608:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948820.406609:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948820.406612:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000010:2.0:1784948820.406616:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000001:3.0:1784948820.406617:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948820.406618:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948820.406620:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948820.406623:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948820.406627:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948820.406631:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948820.406632:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.406638:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948820.406639:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948820.406641:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784948820.406643:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000800:00000001:0.0:1784948820.406650:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948820.406652:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:2.0:1784948820.406653:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000020:00000001:1.0:1784948820.406657:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948820.406657:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 00000020:00000001:1.0:1784948820.406665:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:0.0:1784948820.406666:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948820.406668:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1784948820.406677:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e164874b x1871651289655040/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:481/0 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948820.406677:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948820.406687:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436800 00000100:00000001:3.0:1784948820.406696:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948820.406704:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948820.406709:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 155152 00000100:00100000:1.0:1784948820.406723:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e164874b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289655040:12345-0@lo:400: Request processed in 933us (1482us total) trans 0 rc 0/0 00000800:00000001:0.0:1784948820.406727:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:3.0:1784948820.406729:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948820.406738:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784948820.406741:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:0.0:1784948820.406742:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948820.406746:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009129b91b. 00000100:00100000:1.0:1784948820.406748:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 712 00000100:00000040:2.0:1784948820.406750:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948820.406751:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784948820.406777:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:3.0:1784948820.406779:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000100:00000001:1.0:1784948820.406786:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000200:0.0:1784948820.406790:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:3.0:1784948820.406792:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000100:00000001:1.0:1784948820.406793:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784948820.406802:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948820.406803:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000100:00000001:2.0:1784948820.406809:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948820.406809:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948820.406810:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000010:1.0:1784948820.406812:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000100:00000200:0.0:1784948820.406819:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000010:1.0:1784948820.406847:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000f7a7941e. 00000100:00000010:0.0:1784948820.406852:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e9bfc541. 00000100:00000200:3.0:1784948820.406852:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948820.406860:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000e9bfc541 x1871651289655296 msgsize 224 00000020:00000040:1.0:1784948820.406861:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:1.0:1784948820.406876:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784948820.406883:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000afc15cec. 00000100:00100000:0.0:1784948820.406884:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:1.0:1784948820.406896:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000200:3.0:1784948820.406919:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948820.406921:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948820.406937:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784948820.406947:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000010:1.0:1784948820.406951:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006be72eea. 00000100:00000001:3.0:1784948820.406954:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.406955:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784948820.406968:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948820.406974:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948820.406989:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948820.406999:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bc3ccb92. 00000100:00000001:3.0:1784948820.407004:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:2.0:1784948820.407008:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:3.0:1784948820.407013:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00100000:2.0:1784948820.407016:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289655296 00000100:00000001:3.0:1784948820.407021:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:2.0:1784948820.407024:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948820.407028:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.407030:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948820.407040:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948820.407040:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948820.407051:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948820.407055:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=142 v=5 (5 5 5 5) 00000100:00100000:2.0:1784948820.407059:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289655296 00000100:00001000:3.0:1784948820.407067:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=142 v=5 (1 1 1 1) 00000020:00000001:2.0:1784948820.407068:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948820.407073:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:3.0:1784948820.407095:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:2.0:1784948820.407097:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948820.407102:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948820.407104:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784948820.407111:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:3.0:1784948820.407111:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:2.0:1784948820.407121:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:3.0:1784948820.407121:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.407134:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948820.407134:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948820.407140:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:2.0:1784948820.407145:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948820.407146:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948820.407150:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000020:00000010:2.0:1784948820.407160:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f1904ca. 00000020:00000010:2.0:1784948820.407168:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000895f7a1b. 00000020:00000010:2.0:1784948820.407174:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000001:3.0:1784948820.407176:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948820.407184:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1784948820.407185:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948820.407193:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948820.407193:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948820.407197:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948820.407204:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948820.407209:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948820.407211:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948820.407215:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784948820.407219:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948820.407234:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000055b3248 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289655040:0@lo:400: 00000100:00000001:2.0:1784948820.407241:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.407245:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.407253:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948820.407256:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784948820.407261:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000055b3248 x1871651289655040/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948820.407262:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948820.407275:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 688 00000100:00000040:2.0:1784948820.407283:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948820.407290:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558294912 : -115544151256704 : ffff96e9c7e89180) 00000100:00000001:3.0:1784948820.407295:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:2.0:1784948820.407300:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e9bfc541 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:481/0 lens 224/0 e 0 to 0 dl 1784948831 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948820.407303:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948820.407310:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 00000100:00000001:2.0:1784948820.407315:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948820.407321:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 02000000:00000001:3.0:1784948820.407322:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1784948820.407325:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948820.407328:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:2.0:1784948820.407330:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000020:00000040:3.0:1784948820.407336:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00100000:2.0:1784948820.407341:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e9bfc541 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289655296:12345-192.168.202.137@tcp:400: 00000020:00000001:3.0:1784948820.407347:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000200:2.0:1784948820.407349:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289655296 02000000:00000010:3.0:1784948820.407354:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 00000020:00000001:2.0:1784948820.407355:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948820.407360:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948820.407364:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948820.407365:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1784948820.407367:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:3.0:1784948820.407374:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:2.0:1784948820.407375:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000010:3.0:1784948820.407381:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000055b3248. 00000020:00000001:2.0:1784948820.407382:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948820.407387:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948820.407392:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784948820.407392:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:2.0:1784948820.407398:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784948820.407400:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948820.407401:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948820.407408:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948820.407411:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948820.407414:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948820.407419:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948820.407422:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000038c09866. 00000100:00000001:3.0:1784948820.407422:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948820.407430:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948820.407431:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.407465:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948820.407470:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948820.407471:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948820.407479:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289655296 00000100:00000001:3.0:1784948820.407480:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948820.407483:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948820.407491:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e9bfc541 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:481/0 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948820.407502:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948820.407505:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948820.407508:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=137 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948820.407512:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948820.407515:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948820.407518:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948820.407521:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948820.407523:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948820.407525:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948820.407528:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948820.407531:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000100:00000200:2.0:1784948820.407535:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289655296, offset 224 00000400:00000200:2.0:1784948820.407540:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948820.407548:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948820.407553:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3281:3281:240:4294967295] 192.168.202.137@tcp LPNI seq info [2124:2124:8:4294967295] 00000400:00000200:2.0:1784948820.407564:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948820.407570:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948820.407573:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d625f2e8. 00000800:00000200:2.0:1784948820.407578:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948820.407585:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784948820.407588:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d625f2e8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948820.407604:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948820.407607:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948820.407610:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948820.407612:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948820.407614:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948820.407617:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e9bfc541 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:481/0 lens 224/224 e 0 to 0 dl 1784948831 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948820.407628:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e9bfc541 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289655296:12345-192.168.202.137@tcp:400: Request processed in 311us (766us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948820.407638:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 688 00000100:00000040:2.0:1784948820.407642:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948820.407645:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948820.407647:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948820.407650:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000895f7a1b. 00000020:00000010:2.0:1784948820.407654:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948820.407658:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f1904ca. 00000020:00000040:2.0:1784948820.407662:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948820.407665:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948820.407701:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948820.407707:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d625f2e8. 00000400:00000200:0.0:1784948820.407714:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948820.407718:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948820.407725:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948820.407731:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000800:00000001:2.0:1784948820.407733:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948820.407734:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000800:00000001:2.0:1784948820.407741:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948820.407743:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784948820.407746:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948820.407747:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948820.407762:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948820.407784:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436800 00000400:00000200:2.0:1784948820.407791:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8475 [1] + 224 00000800:00000001:2.0:1784948820.407796:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948820.407801:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948820.407803:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948820.407807:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948820.407812:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948820.407815:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948820.407825:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948820.407845:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948820.407851:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948820.407854:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.407944:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948820.407950:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948820.407958:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948820.407960:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948820.407964:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000400:00000200:3.0:1784948820.407973:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948820.407977:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000001:3.0:1784948820.407982:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948820.407990:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948820.408004:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948820.408017:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948820.408019:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.408023:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948820.408026:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948820.408028:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948820.408031:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948820.408034:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948820.408042:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=137 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948820.408048:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=137 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948820.408052:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948820.408054:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948820.408057:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948820.408063:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948820.408088:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948820.408093:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948820.408096:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948820.408099:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948820.408101:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948820.408106:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948820.408118:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948820.408122:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948820.408124:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948820.408130:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948820.408143:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b13d185 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289655296:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948820.408150:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.408153:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948820.408158:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b13d185 x1871651289655296/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948820.408170:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948820.408173:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948820.408176:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948820.408182:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948820.408184:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948820.408186:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948820.408190:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948820.408193:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:3.0:1784948820.408197:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948820.408199:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948820.408201:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b13d185. 00000100:00000001:3.0:1784948820.408206:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948820.408208:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.408211:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948820.408218:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948820.408220:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948820.408223:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948820.408226:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948820.449033:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305759788, at 4305759789 00000100:00000040:0.0:1784948820.449044:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305759788, at 4305759789 00000100:00000010:3.0:1784948820.449049:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f68fd7be. 00000100:00000010:0.0:1784948820.449059:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000040:1.0:1784948820.449066:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305759788, at 4305759789 00000100:00000040:2.0:1784948820.449077:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305759788, at 4305759789 00000100:00000010:1.0:1784948820.449084:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000824e9543. 00000100:00000010:2.0:1784948820.449087:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000abbd7d53. 00010000:00000001:2.0:1784948821.236082:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948821.236094:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948821.236106:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948821.236109:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948821.236232:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948821.236238:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948821.236243:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948822.260084:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948822.260096:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948822.260107:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948822.260110:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948822.260238:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948822.260247:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948822.260252:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948823.284052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948823.284062:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948823.284073:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948823.284076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948823.284178:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948823.284187:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948823.284193:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948824.308131:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948824.308147:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948824.308161:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948824.308165:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948824.308320:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948824.308328:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948824.308333:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948825.332054:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948825.332068:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948825.332078:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948825.332081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948825.332191:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948825.332199:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948825.332204:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948825.524155:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1784948825.524178:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948825.524183:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948825.524190:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948825.524203:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948825.524205:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948825.524216:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1784948825.524222:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 00000100:00000001:2.0:1784948825.524224:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.524232:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948825.524240:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948825.524252:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948825.524258:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948825.524271:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948825.524287:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002aeee8e9. 00000100:00000001:0.0:1784948825.524299:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948825.524314:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948825.524331:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948825.524335:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948825.524341:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59e6b47] to pc [ptlrpcd_00_03+3] req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948825.524379:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948825.524382:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948825.524388:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948825.524392:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948825.524397:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948825.524433:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ee0ebd63. 00000020:00000040:0.0:1784948825.524439:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948825.524443:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948825.524446:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948825.524450:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948825.524455:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000eafb2cdc. 00000100:00000001:3.0:1784948825.524459:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948825.524461:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.524468:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784948825.524471:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000040:0.0:1784948825.524477:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948825.524479:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948825.524480:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.524495:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948825.524501:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948825.524502:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948825.524509:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948825.524511:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948825.524516:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948825.524521:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948825.524522:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0ebd63] to pc [ptlrpcd_00_00+0] req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948825.524533:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948825.524552:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948825.524555:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948825.524561:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948825.524565:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948825.524567:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948825.524569:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948825.524584:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11102) 00000100:00100000:3.0:1784948825.524587:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289655424:0@lo:400: 00000100:00000001:3.0:1784948825.524597:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948825.524607:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948825.524611:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948825.524616:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948825.524621:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784948825.524626:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 00000100:00100000:2.0:1784948825.524647:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 02000000:00000001:3.0:1784948825.524648:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.524656:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948825.524657:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784948825.524660:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4db7024. 00000100:00000001:1.0:1784948825.524666:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.524666:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948825.524672:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948825.524672:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000001:2.0:1784948825.524680:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948825.524683:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:3.0:1784948825.524685:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289655424, portal 25 00000100:00000001:2.0:1784948825.524686:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948825.524690:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948825.524693:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558303872 : -115544151247744 : ffff96e9c7e8b480) 00000100:00000040:2.0:1784948825.524699:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948825.524710:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948825.524755:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948825.524761:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784948825.524764:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:3.0:1784948825.524765:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000100:00000001:2.0:1784948825.524769:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948825.524770:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289655424, offset 0 02000000:00000001:2.0:1784948825.524780:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784948825.524784:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784948825.524787:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948825.524793:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948825.524797:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ee0ebd63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289655552:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948825.524804:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784948825.524806:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784948825.524810:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948825.524815:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948825.524818:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948825.524822:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436880 02000000:00000010:2.0:1784948825.524825:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000f1904ca. 02000000:00000001:2.0:1784948825.524831:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948825.524834:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7a65 [2] + 0 00000400:00000010:3.0:1784948825.524839:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000054d5ba64. 00000400:00000010:2.0:1784948825.524842:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000080bb5714. 00000400:00000200:3.0:1784948825.524846:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006b55db80 00000400:00000010:2.0:1784948825.524855:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000200:3.0:1784948825.524859:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784948825.524862:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289655552, portal 10 00000400:00000200:3.0:1784948825.524867:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948825.524869:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948825.524872:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558293120 : -115544151258496 : ffff96e9c7e88a80) 00000400:00000200:3.0:1784948825.524875:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006b55db80 00000400:00000010:3.0:1784948825.524879:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000006b55db80. 00000100:00000040:2.0:1784948825.524882:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948825.524887:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948825.524892:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:2.0:1784948825.524896:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948825.524899:0:337121:0:(events.c:337:request_in_callback()) incoming req@000000000ae6b462 x1871651289655424 msgsize 224 00000100:00000040:2.0:1784948825.524903:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948825.524908:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000100:00000200:2.0:1784948825.524911:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289655552, offset 0 00000400:00000200:2.0:1784948825.524917:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:3.0:1784948825.524931:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784948825.524933:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784948825.524943:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948825.524954:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000400:00000010:3.0:1784948825.524956:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000100:00000001:3.0:1784948825.524960:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784948825.524964:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000200:3.0:1784948825.524967:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948825.524974:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948825.524984:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948825.524985:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:3.0:1784948825.524991:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948825.524994:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948825.525004:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784948825.525006:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948825.525013:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3282:3282:240:4294967295] 192.168.202.137@tcp LPNI seq info [2125:2125:8:4294967295] 00000100:00000001:3.0:1784948825.525013:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784948825.525030:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948825.525031:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948825.525037:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289655424 00000800:00000200:2.0:1784948825.525041:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784948825.525042:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948825.525042:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948825.525048:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948825.525051:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948825.525052:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305764907 00000800:00000010:2.0:1784948825.525053:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006aefc2f9. 02000000:00000001:0.0:1784948825.525058:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948825.525063:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00100000:0.0:1784948825.525065:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289655424 00000800:00000200:2.0:1784948825.525071:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:0.0:1784948825.525072:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948825.525076:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000040:0.0:1784948825.525078:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784948825.525083:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948825.525088:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948825.525088:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305764907 00000800:00000200:2.0:1784948825.525089:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:0.0:1784948825.525096:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784948825.525098:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000800:00000200:2.0:1784948825.525099:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000006aefc2f9 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:0.0:1784948825.525100:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948825.525106:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784948825.525110:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948825.525115:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305764907 00000020:00000010:0.0:1784948825.525118:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:2.0:1784948825.525121:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948825.525122:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784948825.525123:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948825.525132:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000100:00000040:2.0:1784948825.525132:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2058 usecs (3 jiffies) 00000100:00000001:3.0:1784948825.525132:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948825.525141:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305764868 00000100:00000001:3.0:1784948825.525143:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305764907 00000020:00000010:0.0:1784948825.525145:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eff166f6. 00000100:00000001:2.0:1784948825.525145:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305764907 00000100:00000010:3.0:1784948825.525149:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:2.0:1784948825.525153:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305764868 00000100:00000001:3.0:1784948825.525155:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000010:0.0:1784948825.525157:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000001fb2b624. 00000100:00000001:2.0:1784948825.525160:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305764907 00000100:00000001:3.0:1784948825.525161:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948825.525167:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784948825.525168:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305764868 00000100:00000001:3.0:1784948825.525168:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.525177:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305764907 00000100:00000001:3.0:1784948825.525178:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948825.525182:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948825.525183:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305764868 00000100:00000001:3.0:1784948825.525185:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948825.525190:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948825.525207:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305764907 00000100:00000001:0.0:1784948825.525211:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948825.525213:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948825.525217:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.525219:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.525227:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948825.525234:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948825.525236:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948825.525281:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948825.525300:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948825.525304:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948825.525312:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 713 00000100:00000040:0.0:1784948825.525318:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948825.525323:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782160976 : -115538927390640 : ffff96eaff467050) 00000100:00000040:0.0:1784948825.525331:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000ae6b462 x1871651289655424/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:486/0 lens 224/0 e 0 to 0 dl 1784948836 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948825.525346:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948825.525351:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:0.0:1784948825.525354:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948825.525359:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000100:00100000:0.0:1784948825.525368:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000ae6b462 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289655424:12345-0@lo:400: 00000100:00000200:0.0:1784948825.525375:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289655424 00000020:00000001:0.0:1784948825.525379:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:2.0:1784948825.525386:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784948825.525387:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000200:1.0:1784948825.525390:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1784948825.525395:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784948825.525429:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006aefc2f9. 00000800:00000001:2.0:1784948825.525436:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948825.525437:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948825.525444:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:1.0:1784948825.525444:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948825.525446:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948825.525454:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948825.525457:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948825.525464:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948825.525471:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:1.0:1784948825.525474:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948825.525476:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:0.0:1784948825.525478:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948825.525487:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948825.525492:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000020:00000001:0.0:1784948825.525493:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948825.525499:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:2.0:1784948825.525503:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436900 00000400:00000010:1.0:1784948825.525505:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 02000000:00000001:0.0:1784948825.525506:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948825.525518:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007f1baa02. 00000100:00000001:1.0:1784948825.525520:0:337113:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1784948825.525525:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948825.525531:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948825.525533:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 155376 00000100:00000200:1.0:1784948825.525538:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948825.525540:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948825.525553:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289655424 00000800:00000001:2.0:1784948825.525555:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:0.0:1784948825.525566:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:2.0:1784948825.525576:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:0.0:1784948825.525580:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000ae6b462 x1871651289655424/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:486/0 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948825.525580:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948825.525584:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784948825.525597:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948825.525605:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948825.525621:0:337114:0:(events.c:286:request_in_callback()) Process entered 00010000:00000001:0.0:1784948825.525628:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000200:2.0:1784948825.525629:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00010000:00000001:0.0:1784948825.525637:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948825.525643:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948825.525643:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000098475f18. 00000100:00001000:0.0:1784948825.525645:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=147 v=5 (1 1 1 1) 00000100:00000040:2.0:1784948825.525655:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000098475f18 x1871651289655552 msgsize 224 00000100:00000001:1.0:1784948825.525656:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948825.525659:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948825.525670:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948825.525681:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00100000:2.0:1784948825.525681:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:0.0:1784948825.525692:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948825.525695:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948825.525700:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948825.525704:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784948825.525711:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948825.525719:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000800:00000001:2.0:1784948825.525720:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784948825.525731:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289655424, offset 224 00000800:00000001:2.0:1784948825.525732:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948825.525754:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948825.525763:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948825.525772:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289655552 00000400:00000200:0.0:1784948825.525777:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:1.0:1784948825.525780:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948825.525787:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948825.525791:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948825.525799:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436880 02000000:00000001:1.0:1784948825.525800:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948825.525813:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289655552 00000400:00000200:0.0:1784948825.525823:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8489 [1] + 224 00000020:00000001:1.0:1784948825.525823:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948825.525830:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948825.525838:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:0.0:1784948825.525842:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1784948825.525844:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948825.525855:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784948825.525857:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:1.0:1784948825.525867:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948825.525873:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784948825.525876:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:0.0:1784948825.525886:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948825.525888:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00000001:1.0:1784948825.525898:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948825.525951:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000020:00000010:1.0:1784948825.525967:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dbce179e. 00000020:00000010:1.0:1784948825.525981:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 00000100:00000040:0.0:1784948825.525984:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948825.525995:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:1.0:1784948825.526016:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948825.526024:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948825.526027:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948825.526029:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948825.526033:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948825.526044:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000100:00000001:1.0:1784948825.526048:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000010:0.0:1784948825.526050:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:1.0:1784948825.526058:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948825.526059:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948825.526065:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948825.526068:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948825.526073:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948825.526079:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784948825.526083:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948825.526086:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948825.526089:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948825.526094:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000ae6b462 x1871651289655424/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:486/0 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948825.526100:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948825.526119:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000ae6b462 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289655424:12345-0@lo:400: Request processed in 768us (1213us total) trans 0 rc 0/0 00000100:00000001:1.0:1784948825.526132:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948825.526140:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948825.526141:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 713 00000100:00000001:1.0:1784948825.526142:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948825.526149:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948825.526153:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948825.526160:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000001:3.0:1784948825.526164:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784948825.526165:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948825.526166:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00100000:1.0:1784948825.526167:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 689 00000100:00000001:2.0:1784948825.526174:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000001:3.0:1784948825.526175:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:0.0:1784948825.526178:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000001fb2b624. 00000100:00000040:1.0:1784948825.526181:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948825.526184:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948825.526185:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000020:00000010:0.0:1784948825.526187:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784948825.526193:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470016128 : -115544239535488 : ffff96e9c2a58a80) 00000100:00000001:2.0:1784948825.526196:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948825.526201:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eff166f6. 00000400:00000200:3.0:1784948825.526202:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948825.526211:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000040:1.0:1784948825.526214:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000098475f18 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:486/0 lens 224/0 e 0 to 0 dl 1784948836 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784948825.526217:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948825.526220:0:337121:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1784948825.526233:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000200:3.0:1784948825.526234:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948825.526241:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007f1baa02. 02000000:00000001:0.0:1784948825.526253:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948825.526259:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948825.526272:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784948825.526275:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:0.0:1784948825.526282:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008946a238. 00000100:00000200:3.0:1784948825.526283:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948825.526287:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000400:00000010:0.0:1784948825.526297:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001d10ae9a. 00000100:00000001:1.0:1784948825.526300:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948825.526313:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000100:00000001:3.0:1784948825.526314:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948825.526321:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.526326:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00100000:1.0:1784948825.526333:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000098475f18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289655552:12345-192.168.202.137@tcp:400: 02000000:00000001:3.0:1784948825.526333:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948825.526339:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948825.526344:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948825.526350:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948825.526360:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=147 v=5 (5 5 5 5) 00000100:00000200:1.0:1784948825.526362:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289655552 00000020:00000001:1.0:1784948825.526374:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00001000:3.0:1784948825.526374:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=147 v=5 (1 1 1 1) 00000020:00000001:1.0:1784948825.526383:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784948825.526384:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948825.526391:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948825.526393:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948825.526397:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:1.0:1784948825.526428:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00010000:00000001:3.0:1784948825.526429:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948825.526439:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:3.0:1784948825.526440:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948825.526450:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948825.526454:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948825.526464:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.526467:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948825.526475:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784948825.526476:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948825.526482:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948825.526485:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:3.0:1784948825.526490:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948825.526493:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948825.526505:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:3.0:1784948825.526514:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289655424:0@lo:400: 00000100:00000001:1.0:1784948825.526516:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948825.526524:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948825.526525:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784948825.526532:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948825.526536:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f59e6b47 x1871651289655424/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784948825.526538:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fed721f2. 02000000:00000001:1.0:1784948825.526549:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948825.526552:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948825.526559:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1784948825.526560:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948825.526563:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 00000020:00000001:1.0:1784948825.526568:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948825.526572:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00010000:00000040:1.0:1784948825.526576:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289655552 00000020:00000001:3.0:1784948825.526578:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948825.526583:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00010000:00000001:1.0:1784948825.526587:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:3.0:1784948825.526589:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948825.526596:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002aeee8e9. 02000000:00000001:3.0:1784948825.526600:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00010000:00000200:1.0:1784948825.526604:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000098475f18 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:486/0 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948825.526605:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948825.526609:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f59e6b47. 00000100:00000001:3.0:1784948825.526616:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948825.526620:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.526628:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.526638:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948825.526643:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948825.526647:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948825.526649:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948825.526656:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948825.526657:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948825.526666:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=142 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948825.526675:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948825.526681:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948825.526687:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948825.526692:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948825.526695:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948825.526700:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948825.526706:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948825.526713:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000200:1.0:1784948825.526720:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289655552, offset 224 00000400:00000200:1.0:1784948825.526732:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948825.526746:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948825.526756:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3283:3283:240:4294967295] 192.168.202.137@tcp LPNI seq info [2126:2126:8:4294967295] 00000400:00000200:1.0:1784948825.526775:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948825.526785:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948825.526793:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000800:00000200:1.0:1784948825.526803:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948825.526814:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948825.526820:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948825.526845:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948825.526851:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948825.526856:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948825.526859:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948825.526864:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948825.526872:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000098475f18 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:486/0 lens 224/224 e 0 to 0 dl 1784948836 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948825.526892:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000098475f18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289655552:12345-192.168.202.137@tcp:400: Request processed in 617us (1235us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948825.526947:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 689 00000100:00000040:1.0:1784948825.526954:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948825.526960:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948825.526964:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948825.526970:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000020:00000010:1.0:1784948825.526980:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948825.526987:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dbce179e. 00000020:00000040:1.0:1784948825.527001:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948825.527007:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948825.527047:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948825.527054:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000400:00000200:2.0:1784948825.527064:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948825.527071:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948825.527076:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:2.0:1784948825.527079:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:2.0:1784948825.527084:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948825.527086:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948825.527093:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948825.527101:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948825.527105:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948825.527109:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948825.527116:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948825.527128:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436900 00000400:00000200:2.0:1784948825.527136:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8491 [1] + 224 00000800:00000001:2.0:1784948825.527144:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948825.527151:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948825.527154:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948825.527160:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948825.527164:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948825.527169:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948825.527191:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948825.527214:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948825.527220:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948825.527223:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.527313:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948825.527319:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948825.527326:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948825.527328:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948825.527331:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000080bb5714. 00000400:00000200:3.0:1784948825.527338:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:3.0:1784948825.527341:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000100:00000001:3.0:1784948825.527345:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948825.527352:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948825.527366:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948825.527377:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948825.527379:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.527381:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948825.527384:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948825.527386:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948825.527387:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948825.527390:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948825.527396:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=142 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948825.527428:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=142 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948825.527431:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948825.527433:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948825.527436:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948825.527441:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948825.527451:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948825.527455:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948825.527457:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948825.527460:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948825.527462:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948825.527465:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948825.527482:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948825.527485:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948825.527487:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948825.527491:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948825.527499:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ee0ebd63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289655552:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948825.527504:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.527506:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948825.527509:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ee0ebd63 x1871651289655552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948825.527516:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948825.527518:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948825.527520:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000f1904ca. 02000000:00000001:3.0:1784948825.527525:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948825.527527:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948825.527528:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948825.527531:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948825.527533:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000eafb2cdc. 02000000:00000001:3.0:1784948825.527537:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948825.527538:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948825.527540:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ee0ebd63. 00000100:00000001:3.0:1784948825.527543:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948825.527545:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.527547:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948825.527552:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948825.527554:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948825.527556:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948825.527559:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948825.568011:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305764907, at 4305764908 00000100:00000010:3.0:1784948825.568027:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000040:1.0:1784948825.568046:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305764907, at 4305764908 00000100:00000040:0.0:1784948825.568060:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305764907, at 4305764908 00000100:00000010:1.0:1784948825.568061:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:0.0:1784948825.568079:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:2.0:1784948825.568139:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305764907, at 4305764908 00000100:00000010:2.0:1784948825.568152:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784948826.356049:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948826.356060:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948826.356069:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948826.356071:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948826.356188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948826.356197:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948826.356201:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948826.548035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948826.548054:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948826.548057:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948826.548063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948826.548067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948826.548072:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948826.548074:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948827.380023:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948827.380032:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948827.380039:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948827.380040:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948827.380129:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948827.380134:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948827.380137:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948827.572029:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948827.572048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948827.572050:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948827.572056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948827.572060:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948827.572065:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948827.572068:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948828.404020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948828.404031:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948828.404040:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948828.404042:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948828.404115:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948828.404121:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948828.404124:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948828.595993:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948828.596010:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948828.596013:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948828.596019:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948828.596024:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948828.596028:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948828.596032:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948829.428131:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948829.428141:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948829.428150:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948829.428152:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948829.428246:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948829.428259:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948829.428265:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948829.620104:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948829.620124:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948829.620127:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948829.620138:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948829.620143:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948829.620149:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948829.620153:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948830.452117:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948830.452134:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948830.452149:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948830.452155:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948830.452160:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948830.452167:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948830.452173:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948830.452177:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948830.452180:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948830.452185:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948830.452295:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948830.452303:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948830.452307:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948830.644085:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948830.644105:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948830.644108:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948830.644115:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948830.644120:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948830.644125:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.644128:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948830.644150:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948830.644160:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948830.644172:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000457a2a3. 00000020:00000040:1.0:1784948830.644181:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948830.644188:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948830.644193:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948830.644202:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948830.644212:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000001:1.0:1784948830.644223:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948830.644237:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948830.644252:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948830.644255:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948830.644261:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000457a2a3] to pc [ptlrpcd_00_01+1] req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948830.644291:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948830.644294:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948830.644299:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948830.644302:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948830.644306:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948830.644315:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948830.644323:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948830.644328:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948830.644338:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784948830.644346:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000acf1d731. 00000020:00000040:1.0:1784948830.644353:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948830.644382:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948830.644386:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784948830.644390:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1784948830.644391:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.644399:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948830.644401:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000004c284093. 00000100:00000001:1.0:1784948830.644408:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948830.644411:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948830.644419:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948830.644419:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1784948830.644430:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948830.644435:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948830.644439:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948830.644444:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000457a2a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289655680:0@lo:400: 00000100:00000001:3.0:1784948830.644445:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948830.644451:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000acf1d731] to pc [ptlrpcd_00_02+2] req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948830.644454:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.644462:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948830.644477:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948830.644485:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948830.644485:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948830.644487:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948830.644492:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784948830.644496:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948830.644496:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948830.644498:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11107) 00000100:00000040:3.0:1784948830.644508:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948830.644509:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001936ad9e. 00000100:00000001:1.0:1784948830.644517:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948830.644519:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948830.644523:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948830.644533:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948830.644536:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948830.644537:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006785ee40. 02000000:00000001:3.0:1784948830.644540:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948830.644548:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948830.644551:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948830.644556:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000acf1d731 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289655808:192.168.202.137@tcp:400: 00000400:00000010:0.0:1784948830.644557:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000100:00000001:3.0:1784948830.644564:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948830.644569:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948830.644571:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948830.644574:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948830.644579:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948830.644582:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948830.644588:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4db7024. 00000400:00000010:3.0:1784948830.644594:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000200:3.0:1784948830.644603:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289655808, portal 10 00000100:00000200:0.0:1784948830.644606:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289655680, portal 25 00000100:00000001:3.0:1784948830.644608:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948830.644611:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549162496 : -115544160389120 : ffff96e9c75d3800) 00000100:00000001:0.0:1784948830.644615:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948830.644620:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529525959168 : -115544183592448 : ffff96e9c5fb2a00) 00000100:00000040:3.0:1784948830.644625:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948830.644634:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948830.644640:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948830.644649:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948830.644655:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000100:00000200:3.0:1784948830.644661:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289655808, offset 0 00000100:00000001:0.0:1784948830.644665:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948830.644679:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:3.0:1784948830.644679:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948830.644689:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 00000100:00000200:0.0:1784948830.644697:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289655680, offset 0 00000400:00000200:3.0:1784948830.644709:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948830.644710:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948830.644722:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948830.644750:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948830.644756:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948830.644760:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948830.644766:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3284:3284:240:4294967295] 192.168.202.137@tcp LPNI seq info [2127:2127:8:4294967295] 00000400:00000200:3.0:1784948830.644783:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948830.644791:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948830.644799:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000400:00000200:0.0:1784948830.644805:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:3.0:1784948830.644815:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784948830.644826:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436980 00000800:00000200:3.0:1784948830.644832:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948830.644840:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1784948830.644846:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7a89 [2] + 0 00000400:00000010:0.0:1784948830.644856:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a51bb92. 00000100:00000001:3.0:1784948830.644857:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948830.644866:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2061 usecs (3 jiffies) 00000400:00000200:0.0:1784948830.644871:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000016d64ec5 00000100:00000001:3.0:1784948830.644873:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305769987 00000100:00000010:3.0:1784948830.644884:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000400:00000200:0.0:1784948830.644890:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948830.644894:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948830.644933:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305769988 00000100:00000010:3.0:1784948830.644939:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000400:00000200:0.0:1784948830.644942:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948830.644944:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948830.644953:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305769988 00000100:00000010:3.0:1784948830.644957:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000400:00000200:0.0:1784948830.644958:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000016d64ec5 00000100:00000001:3.0:1784948830.644961:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000400:00000010:0.0:1784948830.644966:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000016d64ec5. 00000100:00000001:3.0:1784948830.644969:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305769988 00000100:00000010:3.0:1784948830.644976:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000176cd51f. 00000100:00000001:0.0:1784948830.644981:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948830.644982:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000200:0.0:1784948830.644989:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784948830.644990:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948830.644997:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948830.645001:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948830.645002:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000003b52fa03 x1871651289655680 msgsize 224 00000100:00000001:3.0:1784948830.645003:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948830.645009:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948830.645013:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:2.0:1784948830.645017:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000100:00000001:3.0:1784948830.645019:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1784948830.645027:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784948830.645028:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784948830.645029:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948830.645034:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948830.645035:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784948830.645041:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948830.645044:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948830.645046:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948830.645054:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000400:00000010:2.0:1784948830.645059:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000100:00000001:0.0:1784948830.645060:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784948830.645066:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948830.645073:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948830.645080:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000400:00000010:0.0:1784948830.645087:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000100:00000001:2.0:1784948830.645088:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948830.645097:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1784948830.645097:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948830.645111:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948830.645111:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948830.645112:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948830.645119:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1784948830.645125:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948830.645132:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948830.645140:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:3.0:1784948830.645142:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948830.645151:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948830.645156:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784948830.645164:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948830.645171:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289655680 02000000:00000001:1.0:1784948830.645177:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:3.0:1784948830.645178:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436a00 00000100:00000001:0.0:1784948830.645182:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948830.645184:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948830.645190:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.645191:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784948830.645192:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 155600 02000000:00000001:1.0:1784948830.645196:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.645206:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948830.645206:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289655680 00000800:00000001:3.0:1784948830.645207:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948830.645215:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948830.645217:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305770027 00000800:00000001:3.0:1784948830.645217:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784948830.645219:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784948830.645226:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784948830.645227:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1784948830.645233:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948830.645234:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305769987 00000400:00000200:3.0:1784948830.645236:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784948830.645240:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784948830.645247:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948830.645248:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784948830.645250:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305770027 00000100:00000200:3.0:1784948830.645252:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784948830.645254:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948830.645261:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305769988 00000100:00000010:3.0:1784948830.645263:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b13d185. 00000020:00000010:1.0:1784948830.645264:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00000040:3.0:1784948830.645274:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000006b13d185 x1871651289655808 msgsize 224 00000100:00000001:0.0:1784948830.645275:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305770027 00000100:00000001:1.0:1784948830.645275:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948830.645285:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000100:00000001:0.0:1784948830.645289:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305769988 00000100:00100000:3.0:1784948830.645293:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948830.645301:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305770027 00000020:00000010:1.0:1784948830.645302:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bf7eadde. 00000100:00000001:0.0:1784948830.645313:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305769988 00000020:00000010:1.0:1784948830.645313:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 00000100:00000001:3.0:1784948830.645315:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948830.645324:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948830.645327:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000800:00000001:3.0:1784948830.645328:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948830.645331:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948830.645338:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948830.645339:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948830.645347:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948830.645348:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948830.645353:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948830.645356:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00100000:2.0:1784948830.645373:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289655808 02000000:00000001:2.0:1784948830.645376:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948830.645379:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948830.645382:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948830.645383:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.645385:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948830.645386:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948830.645390:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.645392:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1784948830.645396:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289655808 00000100:00000001:0.0:1784948830.645400:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948830.645401:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948830.645402:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.645408:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1784948830.645408:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784948830.645414:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948830.645416:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948830.645418:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:1.0:1784948830.645419:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948830.645423:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948830.645427:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948830.645432:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948830.645437:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948830.645442:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:1.0:1784948830.645443:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948830.645446:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948830.645448:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000010:2.0:1784948830.645452:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00100000:1.0:1784948830.645459:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 714 00000020:00000010:2.0:1784948830.645460:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000100:00000040:1.0:1784948830.645467:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000020:00000010:2.0:1784948830.645468:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000895f7a1b. 00000100:00000001:1.0:1784948830.645474:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782157904 : -115538927393712 : ffff96eaff466450) 00000020:00000010:2.0:1784948830.645477:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000040:1.0:1784948830.645482:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003b52fa03 x1871651289655680/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:491/0 lens 224/0 e 0 to 0 dl 1784948841 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948830.645493:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948830.645494:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948830.645500:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000010:1.0:1784948830.645501:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00000001:2.0:1784948830.645503:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948830.645506:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948830.645508:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948830.645511:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000100:00000001:2.0:1784948830.645517:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00100000:1.0:1784948830.645519:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003b52fa03 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289655680:12345-0@lo:400: 00000100:00000001:2.0:1784948830.645523:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:1.0:1784948830.645526:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289655680 00000100:00000001:2.0:1784948830.645528:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948830.645531:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948830.645537:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948830.645540:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948830.645542:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948830.645544:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948830.645548:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:2.0:1784948830.645549:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948830.645550:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948830.645559:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948830.645562:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948830.645565:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.645568:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000001:1.0:1784948830.645569:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948830.645573:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948830.645575:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948830.645581:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:2.0:1784948830.645585:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 690 00000100:00000001:3.0:1784948830.645588:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000010:1.0:1784948830.645589:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000afc15cec. 00000100:00000040:2.0:1784948830.645590:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948830.645594:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:1.0:1784948830.645597:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.645598:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482526080 : -115544227025536 : ffff96e9c3646d80) 00000100:00000001:3.0:1784948830.645599:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948830.645605:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948830.645608:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948830.645609:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948830.645613:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006b13d185 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:491/0 lens 224/0 e 0 to 0 dl 1784948841 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1784948830.645620:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289655680 00010000:00000001:1.0:1784948830.645627:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784948830.645633:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000200:1.0:1784948830.645637:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003b52fa03 x1871651289655680/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:491/0 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948830.645640:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:2.0:1784948830.645648:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948830.645656:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00010000:00000001:1.0:1784948830.645659:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948830.645662:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948830.645666:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006b13d185 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289655808:12345-192.168.202.137@tcp:400: 00000100:00001000:1.0:1784948830.645670:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=152 v=5 (1 1 1 1) 00000100:00000200:2.0:1784948830.645675:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289655808 00000100:00000001:1.0:1784948830.645680:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1784948830.645681:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000040:1.0:1784948830.645688:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000001:2.0:1784948830.645689:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948830.645694:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948830.645695:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:2.0:1784948830.645701:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:1.0:1784948830.645702:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948830.645707:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948830.645707:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784948830.645715:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1784948830.645716:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000040:1.0:1784948830.645721:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:2.0:1784948830.645722:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948830.645726:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000020:00000001:2.0:1784948830.645727:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948830.645733:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:1.0:1784948830.645735:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289655680, offset 224 00000020:00000001:2.0:1784948830.645736:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948830.645741:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:1.0:1784948830.645745:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948830.645748:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948830.645752:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:1.0:1784948830.645758:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:2.0:1784948830.645760:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f09655c0. 02000000:00000001:2.0:1784948830.645764:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.645767:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948830.645770:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436980 00000020:00000001:2.0:1784948830.645773:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948830.645778:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289655808 00000400:00000200:1.0:1784948830.645782:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x84a9 [1] + 224 00010000:00000001:2.0:1784948830.645784:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:1.0:1784948830.645792:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:2.0:1784948830.645793:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006b13d185 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:491/0 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948830.645802:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948830.645812:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:2.0:1784948830.645816:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000200:1.0:1784948830.645820:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948830.645822:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948830.645830:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=147 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948830.645841:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948830.645847:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000040:1.0:1784948830.645849:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948830.645852:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948830.645858:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948830.645861:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.645867:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948830.645875:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948830.645880:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000100:00000200:2.0:1784948830.645883:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289655808, offset 224 00000100:00000001:1.0:1784948830.645885:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784948830.645892:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948830.645901:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948830.645938:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3285:3285:240:4294967295] 192.168.202.137@tcp LPNI seq info [2128:2128:8:4294967295] 00000400:00000200:2.0:1784948830.645952:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948830.645955:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1784948830.645961:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948830.645962:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000100:00000001:0.0:1784948830.645963:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:1.0:1784948830.645970:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000800:00000010:2.0:1784948830.645971:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006d8f9877. 00000100:00000001:1.0:1784948830.645976:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948830.645980:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000200:2.0:1784948830.645982:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000001:0.0:1784948830.645985:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784948830.645986:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784948830.645993:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:2.0:1784948830.645994:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:1.0:1784948830.645995:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000010:0.0:1784948830.646002:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006785ee40. 00010000:00000001:1.0:1784948830.646004:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000800:00000200:2.0:1784948830.646004:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000006d8f9877 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:1.0:1784948830.646009:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948830.646015:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000020:00000001:1.0:1784948830.646015:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:0.0:1784948830.646022:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000040:1.0:1784948830.646023:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003b52fa03 x1871651289655680/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:491/0 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948830.646026:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.646035:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1784948830.646039:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948830.646047:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:0.0:1784948830.646048:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948830.646051:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948830.646057:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003b52fa03 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289655680:12345-0@lo:400: Request processed in 556us (1048us total) trans 0 rc 0/0 00000020:00000001:2.0:1784948830.646059:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948830.646070:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b13d185 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:491/0 lens 224/224 e 0 to 0 dl 1784948841 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948830.646084:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 714 00000100:00000040:1.0:1784948830.646095:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948830.646102:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:0.0:1784948830.646107:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948830.646108:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948830.646116:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000020:00000010:1.0:1784948830.646126:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000800:00000200:3.0:1784948830.646136:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:1.0:1784948830.646137:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bf7eadde. 00000100:00100000:2.0:1784948830.646145:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006b13d185 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289655808:12345-192.168.202.137@tcp:400: Request processed in 505us (864us total) trans 0 rc 0/0 00000800:00000010:3.0:1784948830.646146:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006d8f9877. 00000020:00000040:1.0:1784948830.646149:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948830.646153:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1784948830.646156:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784948830.646160:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784948830.646161:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784948830.646166:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000afc15cec. 00000100:00000001:0.0:1784948830.646172:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948830.646175:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00100000:2.0:1784948830.646176:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 690 00000400:00000200:3.0:1784948830.646176:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:0.0:1784948830.646180:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948830.646182:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948830.646187:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948830.646188:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:3.0:1784948830.646190:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000100:00000200:1.0:1784948830.646191:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948830.646196:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.646196:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:3.0:1784948830.646197:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000400:00000010:1.0:1784948830.646200:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b743d1b9. 00000100:00000001:2.0:1784948830.646202:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:0.0:1784948830.646206:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948830.646209:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000010:2.0:1784948830.646210:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000895f7a1b. 00000400:00000010:1.0:1784948830.646211:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bed828fe. 00000100:00000001:3.0:1784948830.646213:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00001000:0.0:1784948830.646216:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=152 v=5 (5 5 5 5) 00000020:00000010:2.0:1784948830.646223:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000100:00001000:0.0:1784948830.646227:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=152 v=5 (1 1 1 1) 00000800:00000001:3.0:1784948830.646227:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784948830.646230:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000800:00000001:3.0:1784948830.646237:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784948830.646240:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948830.646241:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000800:00000001:3.0:1784948830.646244:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948830.646248:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.646249:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948830.646252:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948830.646259:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948830.646266:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948830.646267:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948830.646275:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948830.646283:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436a00 00000100:00000040:0.0:1784948830.646287:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948830.646313:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x84a5 [1] + 224 00000800:00000001:3.0:1784948830.646332:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948830.646341:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948830.646348:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000001:3.0:1784948830.646350:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948830.646353:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948830.646382:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948830.646408:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:0.0:1784948830.646411:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948830.646414:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948830.646424:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948830.646449:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000457a2a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289655680:0@lo:400: 00000100:00000040:3.0:1784948830.646455:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948830.646466:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948830.646476:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948830.646481:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000457a2a3 x1871651289655680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948830.646487:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948830.646496:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948830.646499:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:3.0:1784948830.646502:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948830.646506:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948830.646512:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001936ad9e. 02000000:00000001:0.0:1784948830.646519:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948830.646523:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948830.646526:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948830.646526:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948830.646534:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784948830.646535:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948830.646542:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 00000400:00000001:2.0:1784948830.646545:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1784948830.646551:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000001:2.0:1784948830.646551:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1784948830.646558:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000010:2.0:1784948830.646558:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000100:00000010:0.0:1784948830.646564:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000457a2a3. 00000400:00000200:2.0:1784948830.646569:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000100:00000001:0.0:1784948830.646575:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000400:00000010:2.0:1784948830.646576:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000001:0.0:1784948830.646581:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948830.646586:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948830.646587:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948830.646597:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:2.0:1784948830.646598:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948830.646602:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948830.646611:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948830.646615:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948830.646628:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948830.646642:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948830.646644:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948830.646648:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948830.646650:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948830.646653:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948830.646655:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948830.646659:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948830.646665:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=147 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948830.646669:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=147 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948830.646672:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948830.646673:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948830.646675:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948830.646679:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948830.646686:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.646690:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948830.646691:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948830.646693:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948830.646694:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948830.646697:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948830.646703:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948830.646705:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948830.646706:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948830.646708:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948830.646715:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000acf1d731 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289655808:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948830.646719:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948830.646721:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948830.646723:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000acf1d731 x1871651289655808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948830.646729:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948830.646731:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948830.646733:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 02000000:00000001:2.0:1784948830.646737:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948830.646738:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948830.646739:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948830.646741:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948830.646743:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000004c284093. 02000000:00000001:2.0:1784948830.646746:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948830.646747:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948830.646749:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000acf1d731. 00000100:00000001:2.0:1784948830.646770:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948830.646771:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948830.646773:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948830.646779:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948830.646782:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948830.646786:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948830.646789:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948830.688020:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305770027, at 4305770028 00000100:00000010:3.0:1784948830.688038:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000176cd51f. 00000100:00000040:2.0:1784948830.688042:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305770027, at 4305770028 00000100:00000040:1.0:1784948830.688058:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305770027, at 4305770028 00000100:00000040:0.0:1784948830.688062:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305770027, at 4305770028 00000100:00000010:2.0:1784948830.688063:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:1.0:1784948830.688075:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:0.0:1784948830.688080:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784948831.476056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948831.476069:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948831.476080:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948831.476081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948831.476237:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948831.476244:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948831.476249:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948831.668010:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948831.668032:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948831.668036:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948831.668044:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948831.668049:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948831.668055:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948831.668059:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948832.500030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948832.500043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948832.500056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948832.500059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948832.500140:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948832.500147:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948832.500152:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948832.692114:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948832.692141:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948832.692145:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948832.692154:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948832.692160:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948832.692166:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948832.692171:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948833.524024:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948833.524033:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948833.524041:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948833.524042:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948833.524203:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948833.524213:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948833.524218:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948833.716081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948833.716100:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948833.716107:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948833.716117:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948833.716122:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948833.716130:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948833.716135:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948834.548107:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948834.548118:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948834.548127:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948834.548129:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948834.548219:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948834.548231:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948834.548239:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948834.740028:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948834.740048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948834.740051:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948834.740059:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948834.740064:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948834.740069:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948834.740072:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948835.572010:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948835.572018:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948835.572025:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948835.572026:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948835.572085:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948835.572091:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948835.572095:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948835.763992:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1784948835.763992:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948835.764002:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948835.764005:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948835.764008:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948835.764011:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784948835.764013:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006f266164. 00000100:00000001:0.0:1784948835.764014:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948835.764018:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948835.764021:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948835.764022:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948835.764030:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948835.764034:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948835.764041:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948835.764049:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000adf08f9b. 00000100:00000001:1.0:1784948835.764056:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948835.764065:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948835.764075:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948835.764077:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948835.764082:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f266164] to pc [ptlrpcd_00_03+3] req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948835.764106:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948835.764108:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948835.764112:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948835.764115:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948835.764118:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948835.764120:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000053d87417. 00000100:00000001:2.0:1784948835.764122:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948835.764123:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948835.764127:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948835.764128:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948835.764128:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784948835.764132:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948835.764132:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1784948835.764138:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e79c6895. 00000100:00000040:2.0:1784948835.764140:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948835.764142:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948835.764151:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948835.764152:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948835.764155:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764160:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784948835.764162:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948835.764163:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784948835.764166:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948835.764167:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764168:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948835.764174:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053d87417] to pc [ptlrpcd_00_00+0] req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948835.764175:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948835.764181:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006f266164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289655936:0@lo:400: 00000100:00000001:2.0:1784948835.764190:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948835.764196:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948835.764199:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948835.764199:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948835.764201:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784948835.764206:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11112) 00000100:00000001:3.0:1784948835.764207:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1784948835.764208:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 00000100:00000001:3.0:1784948835.764212:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1784948835.764213:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764217:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:2.0:1784948835.764223:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a16404b8. 00000100:00000040:3.0:1784948835.764227:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948835.764235:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000100:00000001:0.0:1784948835.764238:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948835.764244:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764244:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948835.764246:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948835.764248:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289655936, portal 25 02000000:00000001:3.0:1784948835.764251:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948835.764253:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1784948835.764255:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948835.764257:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549160704 : -115544160390912 : ffff96e9c75d3100) 02000000:00000001:3.0:1784948835.764259:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948835.764265:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000053d87417 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289656064:192.168.202.137@tcp:400: 00000100:00000040:2.0:1784948835.764269:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.764279:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948835.764283:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948835.764286:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948835.764288:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784948835.764290:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784948835.764295:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000010:3.0:1784948835.764297:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948835.764300:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948835.764301:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000400:00000010:3.0:1784948835.764305:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000100:00000200:2.0:1784948835.764306:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289655936, offset 0 00000400:00000010:3.0:1784948835.764329:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000100:00000200:3.0:1784948835.764333:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289656064, portal 10 00000100:00000001:3.0:1784948835.764335:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948835.764336:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549159808 : -115544160391808 : ffff96e9c75d2d80) 00000400:00000200:2.0:1784948835.764341:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:3.0:1784948835.764344:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.764351:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948835.764353:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948835.764355:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000200:3.0:1784948835.764357:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289656064, offset 0 00000400:00000200:2.0:1784948835.764361:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948835.764366:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948835.764378:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436a80 00000400:00000200:3.0:1784948835.764384:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948835.764386:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7b25 [2] + 0 00000400:00000010:2.0:1784948835.764392:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bb87e206. 00000400:00000200:3.0:1784948835.764393:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948835.764399:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948835.764401:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000656af8c6 00000400:00000200:3.0:1784948835.764404:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948835.764408:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784948835.764411:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948835.764411:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3286:3286:240:4294967295] 192.168.202.137@tcp LPNI seq info [2129:2129:8:4294967295] 00000400:00000200:2.0:1784948835.764420:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948835.764431:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948835.764437:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948835.764438:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000656af8c6 00000400:00000010:2.0:1784948835.764441:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000656af8c6. 00000800:00000010:3.0:1784948835.764443:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000100:00000001:2.0:1784948835.764447:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948835.764450:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:3.0:1784948835.764455:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:2.0:1784948835.764456:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000e3680464 x1871651289655936 msgsize 224 00000100:00100000:2.0:1784948835.764461:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948835.764463:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:3.0:1784948835.764467:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948835.764471:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948835.764474:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784948835.764482:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000010:2.0:1784948835.764483:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000100:00000001:2.0:1784948835.764486:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948835.764490:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.764490:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948835.764501:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2061 usecs (3 jiffies) 00000100:00000001:2.0:1784948835.764503:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948835.764509:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.764510:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305775107 00000100:00000001:2.0:1784948835.764520:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948835.764522:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000176cd51f. 00000100:00000001:0.0:1784948835.764523:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948835.764523:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784948835.764529:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948835.764530:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289655936 00000100:00000001:2.0:1784948835.764534:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305775146 02000000:00000001:0.0:1784948835.764535:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948835.764536:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948835.764541:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948835.764545:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764548:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305775107 02000000:00000001:0.0:1784948835.764550:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948835.764551:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305775107 00000100:00000010:3.0:1784948835.764555:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000393cfc50. 00000100:00100000:0.0:1784948835.764557:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289655936 00000100:00000001:2.0:1784948835.764558:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305775146 00000100:00000001:3.0:1784948835.764562:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000001:0.0:1784948835.764565:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948835.764568:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305775107 00000020:00000040:0.0:1784948835.764569:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784948835.764570:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305775107 00000100:00000010:3.0:1784948835.764575:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003e9ea21c. 00000020:00000001:0.0:1784948835.764576:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948835.764577:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305775146 00000020:00000040:0.0:1784948835.764583:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948835.764584:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000001:0.0:1784948835.764590:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948835.764592:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305775107 00000100:00000001:2.0:1784948835.764595:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305775107 00000020:00000001:0.0:1784948835.764596:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784948835.764598:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:2.0:1784948835.764601:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305775146 00000100:00000001:3.0:1784948835.764606:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948835.764607:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948835.764612:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948835.764613:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305775107 00000020:00000010:0.0:1784948835.764615:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784948835.764619:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948835.764621:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948835.764621:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948835.764626:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764626:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948835.764627:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1784948835.764629:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:2.0:1784948835.764633:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764633:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784948835.764634:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000020:00000010:0.0:1784948835.764640:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001936ad9e. 00000100:00000001:2.0:1784948835.764640:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948835.764643:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948835.764643:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948835.764649:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948835.764649:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:0.0:1784948835.764652:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008476ce2e. 00000100:00000001:2.0:1784948835.764654:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:1.0:1784948835.764655:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948835.764656:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948835.764660:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784948835.764661:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784948835.764661:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948835.764663:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000100:00000001:2.0:1784948835.764663:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948835.764666:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.764666:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948835.764668:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:3.0:1784948835.764674:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948835.764675:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948835.764676:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784948835.764678:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000200:1.0:1784948835.764683:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948835.764685:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948835.764690:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948835.764694:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948835.764695:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948835.764698:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948835.764705:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948835.764707:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948835.764709:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948835.764710:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948835.764724:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948835.764728:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948835.764731:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948835.764734:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948835.764738:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948835.764746:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436b00 00000100:00000001:0.0:1784948835.764753:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784948835.764753:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 155824 00000100:00000001:0.0:1784948835.764760:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:2.0:1784948835.764761:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:0.0:1784948835.764771:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 715 00000800:00000001:2.0:1784948835.764771:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948835.764774:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784948835.764777:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948835.764779:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784948835.764780:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:2.0:1784948835.764784:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948835.764784:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948835.764786:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782162000 : -115538927389616 : ffff96eaff467450) 00000100:00000001:2.0:1784948835.764790:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948835.764791:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948835.764792:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:0.0:1784948835.764796:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e3680464 x1871651289655936/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:496/0 lens 224/0 e 0 to 0 dl 1784948846 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784948835.764799:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a2eb6ac4. 00000100:00000040:2.0:1784948835.764803:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000a2eb6ac4 x1871651289656064 msgsize 224 00000100:00100000:2.0:1784948835.764815:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948835.764822:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948835.764826:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784948835.764828:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948835.764830:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000010:0.0:1784948835.764831:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000800:00000001:2.0:1784948835.764836:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:0.0:1784948835.764840:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e3680464 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289655936:12345-0@lo:400: 00000800:00000001:2.0:1784948835.764841:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948835.764848:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289655936 00000020:00000001:0.0:1784948835.764854:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948835.764859:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948835.764863:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948835.764866:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948835.764869:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948835.764873:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948835.764877:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948835.764877:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948835.764881:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289656064 00000020:00000001:0.0:1784948835.764883:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000001:1.0:1784948835.764885:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948835.764887:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:0.0:1784948835.764889:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948835.764889:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948835.764894:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948835.764895:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948835.764897:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289656064 00000020:00000001:1.0:1784948835.764901:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:0.0:1784948835.764902:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000040:1.0:1784948835.764922:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948835.764924:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948835.764926:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948835.764929:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948835.764932:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948835.764937:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948835.764938:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:0.0:1784948835.764942:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:1.0:1784948835.764945:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948835.764949:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1784948835.764950:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007f1baa02. 00000020:00000010:1.0:1784948835.764952:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 02000000:00000001:0.0:1784948835.764956:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948835.764960:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948835.764963:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948835.764964:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 00010000:00000040:0.0:1784948835.764967:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289655936 00000020:00000010:1.0:1784948835.764970:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 00010000:00000001:0.0:1784948835.764973:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000010:1.0:1784948835.764977:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00010000:00000200:0.0:1784948835.764980:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e3680464 x1871651289655936/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:496/0 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948835.764987:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948835.764992:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948835.764995:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948835.764999:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948835.765005:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784948835.765008:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00010000:00000001:0.0:1784948835.765010:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948835.765011:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948835.765014:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948835.765017:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=7 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948835.765024:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784948835.765027:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948835.765028:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948835.765034:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948835.765038:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784948835.765038:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:0.0:1784948835.765041:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948835.765041:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948835.765047:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1784948835.765050:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 691 00000100:00000001:2.0:1784948835.765052:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784948835.765056:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:1.0:1784948835.765057:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948835.765057:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948835.765061:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948835.765062:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000100:00000001:1.0:1784948835.765062:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470021504 : -115544239530112 : ffff96e9c2a59f80) 00000100:00000001:2.0:1784948835.765065:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948835.765071:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289655936, offset 224 00000100:00000040:1.0:1784948835.765071:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a2eb6ac4 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:496/0 lens 224/0 e 0 to 0 dl 1784948846 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948835.765084:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784948835.765085:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:1.0:1784948835.765089:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948835.765092:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948835.765095:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000400:00000200:0.0:1784948835.765098:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:1.0:1784948835.765103:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a2eb6ac4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289656064:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948835.765107:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289656064 00000020:00000001:1.0:1784948835.765109:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948835.765111:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948835.765112:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948835.765114:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948835.765115:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948835.765118:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:0.0:1784948835.765120:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436a80 00000020:00000001:1.0:1784948835.765121:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948835.765126:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948835.765128:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948835.765130:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948835.765131:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x84c1 [1] + 224 00000020:00000001:1.0:1784948835.765133:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948835.765137:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948835.765139:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:0.0:1784948835.765141:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1784948835.765143:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006107fe19. 02000000:00000001:1.0:1784948835.765146:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948835.765150:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948835.765150:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948835.765154:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948835.765158:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289656064 00000100:00000001:0.0:1784948835.765159:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:1.0:1784948835.765160:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948835.765168:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a2eb6ac4 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:496/0 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948835.765169:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948835.765186:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948835.765189:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948835.765192:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=152 v=5 (1 1 1 1) 00000100:00000040:0.0:1784948835.765196:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948835.765197:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948835.765200:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948835.765203:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948835.765208:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948835.765211:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948835.765215:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948835.765219:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948835.765222:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784948835.765224:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289656064, offset 224 00000100:00000001:0.0:1784948835.765229:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948835.765230:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948835.765238:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948835.765242:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3287:3287:240:4294967295] 192.168.202.137@tcp LPNI seq info [2130:2130:8:4294967295] 00000400:00000200:1.0:1784948835.765251:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948835.765255:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948835.765258:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000100:00000001:2.0:1784948835.765259:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784948835.765260:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000100:00000001:2.0:1784948835.765264:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:1.0:1784948835.765265:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:0.0:1784948835.765266:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:0.0:1784948835.765274:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000200:1.0:1784948835.765274:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:0.0:1784948835.765278:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:2.0:1784948835.765278:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1784948835.765279:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000400:00000001:2.0:1784948835.765284:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784948835.765285:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948835.765288:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a16404b8. 00000100:00000040:0.0:1784948835.765291:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:2.0:1784948835.765295:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00010000:00000001:0.0:1784948835.765297:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784948835.765300:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948835.765301:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948835.765301:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000100:00000040:1.0:1784948835.765306:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000020:00000001:0.0:1784948835.765308:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784948835.765309:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:1.0:1784948835.765310:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948835.765329:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948835.765332:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:2.0:1784948835.765333:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948835.765334:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e3680464 x1871651289655936/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:496/0 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948835.765339:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a2eb6ac4 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:496/0 lens 224/224 e 0 to 0 dl 1784948846 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948835.765364:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948835.765379:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e3680464 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289655936:12345-0@lo:400: Request processed in 551us (916us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948835.765380:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a2eb6ac4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289656064:12345-192.168.202.137@tcp:400: Request processed in 290us (570us total) trans 0 rc 0/0 00000100:00000001:2.0:1784948835.765380:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948835.765385:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948835.765391:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948835.765395:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948835.765397:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948835.765399:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948835.765403:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 691 02000000:00000001:2.0:1784948835.765403:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948835.765405:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 715 00000100:00000040:1.0:1784948835.765411:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000040:0.0:1784948835.765412:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00001000:2.0:1784948835.765412:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=7 v=5 (5 5 5 5) 00000100:00000001:1.0:1784948835.765415:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948835.765419:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784948835.765420:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00001000:2.0:1784948835.765421:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=7 v=5 (1 1 1 1) 00000020:00000010:1.0:1784948835.765425:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000100:00000001:0.0:1784948835.765427:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784948835.765427:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000800:00000200:3.0:1784948835.765427:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948835.765430:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948835.765431:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:0.0:1784948835.765433:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008476ce2e. 00000800:00000010:3.0:1784948835.765434:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00010000:00000001:2.0:1784948835.765436:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:1.0:1784948835.765437:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00010000:00000001:2.0:1784948835.765441:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948835.765443:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000040:1.0:1784948835.765445:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948835.765446:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948835.765446:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:0.0:1784948835.765450:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001936ad9e. 00000100:00000001:1.0:1784948835.765451:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948835.765455:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784948835.765459:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:3.0:1784948835.765460:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:0.0:1784948835.765468:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000400:00000200:3.0:1784948835.765471:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 02000000:00000010:0.0:1784948835.765473:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007f1baa02. 00000100:00000001:2.0:1784948835.765473:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000001:1.0:1784948835.765475:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784948835.765476:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:2.0:1784948835.765479:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1784948835.765481:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784948835.765483:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948835.765484:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948835.765485:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948835.765485:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784948835.765490:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784948835.765491:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.765491:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000200:0.0:1784948835.765495:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000800:00000001:1.0:1784948835.765495:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948835.765503:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006785ee40. 00000100:00100000:2.0:1784948835.765506:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006f266164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289655936:0@lo:400: 00000400:00000200:1.0:1784948835.765508:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:0.0:1784948835.765512:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000efb93606. 00000100:00000001:2.0:1784948835.765518:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948835.765523:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436b00 00000100:00000001:2.0:1784948835.765523:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948835.765528:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006f266164 x1871651289655936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948835.765533:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x84c9 [1] + 224 00000800:00000001:1.0:1784948835.765543:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948835.765544:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948835.765547:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948835.765550:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:2.0:1784948835.765552:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948835.765554:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000800:00000001:1.0:1784948835.765556:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784948835.765556:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:1.0:1784948835.765560:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784948835.765560:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948835.765563:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000adf08f9b. 00000400:00000200:1.0:1784948835.765567:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:2.0:1784948835.765569:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1784948835.765571:0:337113:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1784948835.765572:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948835.765576:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006f266164. 00000100:00000200:1.0:1784948835.765579:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948835.765581:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948835.765583:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948835.765587:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948835.765592:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948835.765594:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948835.765596:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948835.765598:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948835.765600:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948835.765616:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948835.765622:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948835.765625:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948835.765652:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948835.765656:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948835.765659:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948835.765661:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948835.765663:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000400:00000200:3.0:1784948835.765670:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000400:00000010:3.0:1784948835.765672:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000100:00000001:3.0:1784948835.765674:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948835.765678:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948835.765688:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.765694:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948835.765696:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948835.765697:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948835.765699:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948835.765700:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948835.765701:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948835.765703:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948835.765706:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=152 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948835.765709:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=152 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948835.765711:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948835.765711:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948835.765713:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948835.765715:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948835.765720:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.765722:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948835.765723:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948835.765725:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948835.765725:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948835.765727:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.765736:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948835.765738:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948835.765739:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948835.765741:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948835.765745:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000053d87417 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289656064:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948835.765748:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948835.765749:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948835.765750:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000053d87417 x1871651289656064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948835.765754:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948835.765755:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948835.765756:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948835.765758:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948835.765759:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948835.765760:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948835.765761:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948835.765762:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e79c6895. 02000000:00000001:3.0:1784948835.765764:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948835.765765:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948835.765766:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000053d87417. 00000100:00000001:3.0:1784948835.765768:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948835.765769:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948835.765771:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948835.765774:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948835.765774:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948835.765776:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948835.765777:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948835.806975:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305775146, at 4305775147 00000100:00000040:2.0:1784948835.806975:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305775146, at 4305775147 00000100:00000040:3.0:1784948835.806993:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305775146, at 4305775147 00000100:00000010:2.0:1784948835.806995:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003e9ea21c. 00000100:00000010:1.0:1784948835.806998:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000393cfc50. 00000100:00000010:3.0:1784948835.807006:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000040:0.0:1784948835.807008:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305775146, at 4305775147 00000100:00000010:0.0:1784948835.807020:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000176cd51f. 00010000:00000001:2.0:1784948836.596016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948836.596030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948836.596040:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948836.596042:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948836.596158:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948836.596169:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948836.596174:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948836.788096:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948836.788118:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948836.788120:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948836.788129:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948836.788133:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948836.788138:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948836.788143:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948836.980109:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:3.0:1784948836.980127:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000800:00000200:3.0:1784948836.980136:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000800:00000200:2.0:1784948836.980516:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784948837.620042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948837.620055:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948837.620066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948837.620069:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948837.620177:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948837.620182:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948837.620186:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948837.811991:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948837.812010:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948837.812014:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948837.812024:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948837.812028:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948837.812034:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948837.812037:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948838.644093:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948838.644110:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948838.644125:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948838.644129:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948838.644277:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948838.644284:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948838.644289:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948838.836105:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948838.836128:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948838.836131:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948838.836138:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948838.836143:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948838.836147:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948838.836151:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948839.668116:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948839.668127:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948839.668136:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948839.668137:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948839.668247:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948839.668253:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948839.668256:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948839.860006:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948839.860027:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948839.860029:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948839.860036:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948839.860040:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948839.860046:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948839.860049:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948840.692043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948840.692057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948840.692067:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948840.692073:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948840.692077:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948840.692081:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948840.692086:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948840.692089:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948840.692092:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948840.692095:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948840.692231:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948840.692238:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948840.692243:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784948840.884195:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948840.884213:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948840.884233:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000020:00000040:0.0:1784948840.884247:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948840.884257:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948840.884258:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948840.884268:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948840.884314:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1784948840.884315:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.884323:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948840.884333:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948840.884338:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:0.0:1784948840.884343:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000006487a147. 00000100:00000001:3.0:1784948840.884345:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.884356:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948840.884363:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948840.884380:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948840.884397:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948840.884401:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948840.884408:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_00_01+1] req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948840.884464:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948840.884467:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948840.884472:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948840.884477:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948840.884482:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948840.884486:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009778919c. 00000100:00000001:3.0:1784948840.884488:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784948840.884490:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948840.884494:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948840.884495:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948840.884500:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948840.884501:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784948840.884505:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948840.884513:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948840.884514:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b51ad697. 00000100:00000001:0.0:1784948840.884523:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948840.884530:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948840.884539:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948840.884542:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.884543:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948840.884556:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948840.884558:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948840.884559:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948840.884570:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948840.884572:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009778919c] to pc [ptlrpcd_00_02+2] req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948840.884575:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948840.884585:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948840.884594:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289656192:0@lo:400: 00000100:00000001:3.0:1784948840.884601:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948840.884609:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948840.884612:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948840.884614:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948840.884616:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784948840.884625:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11117) 02000000:00000010:3.0:1784948840.884627:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948840.884633:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948840.884653:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000100:00000001:2.0:1784948840.884665:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784948840.884665:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000100:00000001:2.0:1784948840.884673:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948840.884676:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000200:3.0:1784948840.884678:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289656192, portal 25 00000100:00000001:1.0:1784948840.884683:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948840.884687:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948840.884687:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948840.884691:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.884694:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000040:3.0:1784948840.884710:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948840.884712:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948840.884718:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948840.884726:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948840.884732:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948840.884733:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948840.884741:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784948840.884742:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948840.884747:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000100:00100000:2.0:1784948840.884750:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000009778919c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289656320:192.168.202.137@tcp:400: 00000100:00000200:3.0:1784948840.884755:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289656192, offset 0 00000100:00000001:2.0:1784948840.884760:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948840.884766:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:3.0:1784948840.884771:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784948840.884772:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948840.884779:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784948840.884787:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcf1bf81. 02000000:00000001:2.0:1784948840.884790:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948840.884797:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a16404b8. 00000400:00000200:3.0:1784948840.884804:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:2.0:1784948840.884809:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000100:00000200:2.0:1784948840.884817:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289656320, portal 10 00000100:00000001:2.0:1784948840.884820:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948840.884823:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558294016 : -115544151257600 : ffff96e9c7e88e00) 00000400:00000200:3.0:1784948840.884827:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436b80 00000100:00000040:2.0:1784948840.884837:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948840.884843:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7b4d [2] + 0 00000400:00000010:3.0:1784948840.884858:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000317b8d47. 00000100:00000001:2.0:1784948840.884867:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948840.884872:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948840.884873:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000095fae736 00000400:00000010:2.0:1784948840.884877:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000100:00000200:2.0:1784948840.884883:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289656320, offset 0 00000400:00000200:3.0:1784948840.884888:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948840.884891:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948840.884900:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948840.884941:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948840.884950:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948840.884957:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948840.884963:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948840.884968:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784948840.884975:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3288:3288:240:4294967295] 192.168.202.137@tcp LPNI seq info [2131:2131:8:4294967295] 00000400:00000200:2.0:1784948840.884992:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948840.885001:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948840.885003:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000095fae736 00000800:00000010:2.0:1784948840.885008:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000016632059. 00000400:00000010:3.0:1784948840.885010:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000095fae736. 00000100:00000001:3.0:1784948840.885021:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:2.0:1784948840.885025:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:3.0:1784948840.885027:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784948840.885036:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000009bc35acb x1871651289656192 msgsize 224 00000800:00000200:2.0:1784948840.885048:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:3.0:1784948840.885050:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784948840.885056:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:2.0:1784948840.885059:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000016632059 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948840.885071:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948840.885084:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000400:00000010:3.0:1784948840.885087:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000100:00000001:2.0:1784948840.885089:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.885092:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784948840.885100:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948840.885101:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2061 usecs (3 jiffies) 00000100:00000001:2.0:1784948840.885112:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305780228 00000100:00000010:2.0:1784948840.885125:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000abbd7d53. 00000100:00000001:3.0:1784948840.885127:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948840.885134:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948840.885140:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948840.885146:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.885152:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948840.885154:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948840.885154:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305780228 00000100:00000001:3.0:1784948840.885159:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948840.885165:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f59314ac. 00000100:00100000:1.0:1784948840.885166:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289656192 00000100:00000001:3.0:1784948840.885166:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305780267 02000000:00000001:1.0:1784948840.885177:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948840.885177:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948840.885182:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305780228 00000100:00000001:1.0:1784948840.885186:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948840.885187:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305780228 00000100:00000001:3.0:1784948840.885192:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305780267 00000100:00000001:1.0:1784948840.885195:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948840.885196:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008cd34e64. 00000100:00000001:3.0:1784948840.885204:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305780228 00000100:00000001:2.0:1784948840.885206:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 02000000:00000001:1.0:1784948840.885208:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.885221:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305780267 00000100:00100000:1.0:1784948840.885222:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289656192 00000100:00000001:2.0:1784948840.885222:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305780228 00000020:00000001:1.0:1784948840.885234:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000010:2.0:1784948840.885234:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000082553b41. 00000100:00000001:3.0:1784948840.885238:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305780228 00000020:00000040:1.0:1784948840.885243:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784948840.885245:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948840.885250:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305780267 00000020:00000001:1.0:1784948840.885254:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948840.885255:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948840.885263:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305780228 00000100:00000001:2.0:1784948840.885264:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948840.885266:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784948840.885270:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.885297:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948840.885299:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948840.885307:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948840.885309:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948840.885311:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948840.885316:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948840.885318:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948840.885322:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948840.885324:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948840.885327:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948840.885329:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948840.885334:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948840.885335:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1784948840.885341:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001690d9bc. 00000100:00000001:3.0:1784948840.885343:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948840.885344:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948840.885356:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948840.885357:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.885358:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:0.0:1784948840.885362:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948840.885364:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1784948840.885369:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001690d9bc. 00000100:00000001:3.0:1784948840.885370:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948840.885373:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000016632059. 00000100:00000001:3.0:1784948840.885380:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948840.885386:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1784948840.885391:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000eac166c7. 00000400:00000200:0.0:1784948840.885402:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:1.0:1784948840.885404:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 00000400:00000200:0.0:1784948840.885411:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000020:00000010:1.0:1784948840.885417:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000400:00000010:0.0:1784948840.885418:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000100:00000001:0.0:1784948840.885431:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:3.0:1784948840.885437:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784948840.885444:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948840.885444:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948840.885455:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000800:00000001:3.0:1784948840.885457:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948840.885463:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000001:3.0:1784948840.885466:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948840.885471:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948840.885473:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948840.885478:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948840.885490:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948840.885494:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948840.885497:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948840.885517:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948840.885518:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436c00 00000100:00000001:0.0:1784948840.885526:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948840.885527:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948840.885536:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 156048 00000800:00000001:3.0:1784948840.885546:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948840.885556:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948840.885559:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784948840.885560:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948840.885569:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784948840.885577:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:1.0:1784948840.885585:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 716 00000100:00000001:2.0:1784948840.885586:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948840.885587:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784948840.885594:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:3.0:1784948840.885595:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:1.0:1784948840.885598:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948840.885601:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948840.885605:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000055b3248. 00000100:00000001:2.0:1784948840.885609:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948840.885610:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782163024 : -115538927388592 : ffff96eaff467850) 00000100:00000040:3.0:1784948840.885612:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000055b3248 x1871651289656320 msgsize 224 00000100:00000040:1.0:1784948840.885631:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009bc35acb x1871651289656192/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:501/0 lens 224/0 e 0 to 0 dl 1784948851 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948840.885632:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784948840.885655:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948840.885656:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000010:1.0:1784948840.885666:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001690d9bc. 00000800:00000001:3.0:1784948840.885671:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948840.885674:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:3.0:1784948840.885678:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948840.885680:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000010:1.0:1784948840.885682:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001690d9bc. 00000100:00100000:2.0:1784948840.885688:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289656320 02000000:00000001:2.0:1784948840.885691:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948840.885694:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948840.885700:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948840.885701:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009bc35acb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289656192:12345-0@lo:400: 02000000:00000001:2.0:1784948840.885709:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948840.885720:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289656192 00000100:00100000:2.0:1784948840.885720:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289656320 00000020:00000001:2.0:1784948840.885727:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948840.885730:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948840.885732:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948840.885733:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948840.885740:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948840.885745:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948840.885746:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948840.885753:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948840.885754:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948840.885758:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:1.0:1784948840.885764:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000010:2.0:1784948840.885766:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000020:00000001:1.0:1784948840.885773:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784948840.885773:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948840.885780:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000020:00000001:1.0:1784948840.885787:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000010:2.0:1784948840.885789:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f1904ca. 00000020:00000001:1.0:1784948840.885795:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948840.885797:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000079fab95b. 00000020:00000010:2.0:1784948840.885806:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000020:00000001:1.0:1784948840.885807:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948840.885814:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:2.0:1784948840.885816:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:1.0:1784948840.885820:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948840.885821:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948840.885825:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:1.0:1784948840.885831:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948840.885832:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948840.885841:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948840.885842:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948840.885849:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:1.0:1784948840.885850:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948840.885851:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948840.885864:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001cc3fd72. 00000100:00000001:2.0:1784948840.885874:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948840.885875:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948840.885886:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948840.885891:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948840.885892:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948840.885896:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000040:1.0:1784948840.885901:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289656192 00000100:00100000:2.0:1784948840.885938:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 692 00000100:00000040:2.0:1784948840.885943:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948840.885949:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482522496 : -115544227029120 : ffff96e9c3645f80) 00010000:00000001:1.0:1784948840.885954:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:2.0:1784948840.885960:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000055b3248 x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:501/0 lens 224/0 e 0 to 0 dl 1784948851 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784948840.885970:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009bc35acb x1871651289656192/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:501/0 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948840.885980:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948840.885985:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:2.0:1784948840.885991:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948840.885998:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00010000:00000001:1.0:1784948840.886007:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00100000:2.0:1784948840.886008:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000055b3248 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289656320:12345-192.168.202.137@tcp:400: 00010000:00000001:1.0:1784948840.886017:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948840.886018:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289656320 00000020:00000001:2.0:1784948840.886024:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00001000:1.0:1784948840.886030:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=12 v=5 (1 1 1 1) 00000020:00000001:2.0:1784948840.886030:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948840.886035:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948840.886042:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948840.886046:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784948840.886048:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1784948840.886049:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948840.886056:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948840.886059:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000001:2.0:1784948840.886061:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948840.886065:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948840.886071:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948840.886072:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:2.0:1784948840.886074:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 02000000:00000001:1.0:1784948840.886082:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784948840.886082:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948840.886089:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:1.0:1784948840.886091:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948840.886094:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d4f75efa. 00000100:00000001:1.0:1784948840.886100:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948840.886100:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948840.886108:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948840.886114:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948840.886116:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00010000:00000040:2.0:1784948840.886118:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289656320 00010000:00000001:2.0:1784948840.886124:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000010:1.0:1784948840.886129:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00010000:00000200:2.0:1784948840.886129:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000055b3248 x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:501/0 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948840.886142:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289656192, offset 224 00010000:00000001:2.0:1784948840.886145:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948840.886151:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948840.886155:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=7 v=5 (1 1 1 1) 00000400:00000200:1.0:1784948840.886163:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948840.886164:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948840.886170:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948840.886174:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000200:1.0:1784948840.886182:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784948840.886182:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948840.886187:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948840.886192:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948840.886196:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948840.886201:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436b80 00000400:00000010:2.0:1784948840.886204:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000200:1.0:1784948840.886213:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x84dd [1] + 224 00000100:00000200:2.0:1784948840.886223:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289656320, offset 224 00000400:00000200:1.0:1784948840.886225:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948840.886237:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948840.886240:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948840.886251:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948840.886258:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3289:3289:240:4294967295] 192.168.202.137@tcp LPNI seq info [2132:2132:8:4294967295] 00000400:00000200:2.0:1784948840.886268:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948840.886320:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948840.886323:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000010:2.0:1784948840.886326:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000230b350e. 00000100:00000200:1.0:1784948840.886338:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784948840.886339:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948840.886355:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948840.886362:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000230b350e type 1, nob 320 niov 1 nkiov 1 00000100:00000040:1.0:1784948840.886379:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948840.886390:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948840.886400:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948840.886406:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000001:2.0:1784948840.886406:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948840.886442:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948840.886447:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000020:00000001:2.0:1784948840.886448:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:1.0:1784948840.886454:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000100:00000001:1.0:1784948840.886461:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:2.0:1784948840.886462:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000055b3248 x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:501/0 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948840.886465:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948840.886473:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948840.886484:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784948840.886491:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948840.886495:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948840.886499:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948840.886500:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00100000:2.0:1784948840.886503:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000055b3248 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289656320:12345-192.168.202.137@tcp:400: Request processed in 514us (881us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948840.886508:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948840.886512:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009bc35acb x1871651289656192/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:501/0 lens 224/224 e 0 to 0 dl 1784948851 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784948840.886521:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948840.886529:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1784948840.886535:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 692 00000400:00000010:0.0:1784948840.886537:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000100:00100000:1.0:1784948840.886547:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009bc35acb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289656192:12345-0@lo:400: Request processed in 888us (1505us total) trans 0 rc 0/0 00000100:00000040:2.0:1784948840.886548:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000400:00000200:0.0:1784948840.886553:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000100:00000001:2.0:1784948840.886557:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:0.0:1784948840.886563:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000100:00000001:2.0:1784948840.886565:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00100000:1.0:1784948840.886572:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 716 00000100:00000001:0.0:1784948840.886574:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:2.0:1784948840.886575:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000079fab95b. 00000100:00000040:1.0:1784948840.886577:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000800:00000200:3.0:1784948840.886582:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948840.886585:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000010:2.0:1784948840.886587:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000100:00000200:0.0:1784948840.886589:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1784948840.886591:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000230b350e. 00000100:00000001:1.0:1784948840.886592:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948840.886597:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f1904ca. 00000020:00000010:1.0:1784948840.886600:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000400:00000200:3.0:1784948840.886602:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1784948840.886611:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000020:00000010:1.0:1784948840.886614:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948840.886621:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948840.886625:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:1.0:1784948840.886626:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000eac166c7. 00000400:00000200:3.0:1784948840.886640:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000020:00000040:1.0:1784948840.886641:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000010:3.0:1784948840.886648:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 02000000:00000001:1.0:1784948840.886653:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948840.886658:0:337114:0:(events.c:386:reply_out_callback()) Process entered 02000000:00000010:1.0:1784948840.886659:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000001cc3fd72. 00000800:00000001:2.0:1784948840.886664:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948840.886665:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 02000000:00000001:1.0:1784948840.886668:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000200:0.0:1784948840.886673:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948840.886675:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948840.886680:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948840.886686:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000800:00000001:2.0:1784948840.886688:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784948840.886697:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bb516699. 00000800:00000001:2.0:1784948840.886698:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948840.886704:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000010:1.0:1784948840.886709:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000017500e72. 00000100:00000001:0.0:1784948840.886711:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948840.886719:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948840.886723:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948840.886731:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000200:2.0:1784948840.886737:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436c00 00000100:00000001:0.0:1784948840.886738:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948840.886747:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948840.886752:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x84e5 [1] + 224 02000000:00000001:0.0:1784948840.886759:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948840.886762:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:0.0:1784948840.886769:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=12 v=5 (5 5 5 5) 00000800:00000001:2.0:1784948840.886774:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948840.886779:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00001000:0.0:1784948840.886780:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=12 v=5 (1 1 1 1) 00000400:00000200:2.0:1784948840.886787:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948840.886795:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948840.886797:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948840.886802:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948840.886806:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948840.886813:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948840.886822:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948840.886824:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948840.886832:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948840.886850:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948840.886858:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948840.886868:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784948840.886869:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948840.886876:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948840.886878:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948840.886885:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948840.886892:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948840.886899:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948840.886928:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948840.886941:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948840.886943:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948840.886946:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a16404b8. 00000400:00000200:3.0:1784948840.886954:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:3.0:1784948840.886958:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000100:00100000:0.0:1784948840.886961:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289656192:0@lo:400: 00000100:00000001:3.0:1784948840.886964:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948840.886975:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948840.886978:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948840.886989:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948840.887004:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b3c18d2b x1871651289656192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948840.887004:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948840.887042:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948840.887046:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948840.887050:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784948840.887051:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948840.887057:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1784948840.887057:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948840.887059:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948840.887063:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948840.887064:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948840.887066:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948840.887076:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00001000:3.0:1784948840.887076:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=7 v=5 (5 5 5 5) 00000020:00000001:0.0:1784948840.887083:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00001000:3.0:1784948840.887085:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=7 v=5 (1 1 1 1) 00000020:00000040:0.0:1784948840.887089:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948840.887092:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948840.887097:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948840.887099:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000001:3.0:1784948840.887101:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000010:0.0:1784948840.887106:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000006487a147. 00010000:00080000:3.0:1784948840.887109:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948840.887117:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948840.887123:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00010000:00000001:3.0:1784948840.887125:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948840.887128:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b3c18d2b. 00000100:00000001:3.0:1784948840.887131:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948840.887137:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948840.887138:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948840.887145:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948840.887146:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948840.887149:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948840.887154:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948840.887156:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948840.887168:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948840.887169:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948840.887176:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.887176:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948840.887180:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948840.887203:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948840.887205:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948840.887211:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948840.887230:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000009778919c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289656320:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948840.887239:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948840.887243:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948840.887248:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000009778919c x1871651289656320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948856 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948840.887261:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948840.887264:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948840.887267:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcf1bf81. 02000000:00000001:3.0:1784948840.887297:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948840.887301:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948840.887320:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948840.887325:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948840.887328:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b51ad697. 02000000:00000001:3.0:1784948840.887332:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948840.887334:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948840.887336:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000009778919c. 00000100:00000001:3.0:1784948840.887341:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948840.887343:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948840.887345:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948840.887350:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948840.887352:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948840.887354:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948840.887356:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948840.928013:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305780267, at 4305780268 00000100:00000040:2.0:1784948840.928026:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305780267, at 4305780268 00000100:00000010:0.0:1784948840.928038:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000abbd7d53. 00000100:00000010:2.0:1784948840.928042:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00000100:00000040:3.0:1784948840.928050:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305780267, at 4305780268 00000100:00000040:1.0:1784948840.928083:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305780267, at 4305780268 00000100:00000010:3.0:1784948840.928084:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000100:00000010:1.0:1784948840.928105:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f59314ac. 00000400:00000010:1.0:1784948841.076028:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002b26853b. 00000400:00000010:1.0:1784948841.076062:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000399be696. 00000400:00000010:1.0:1784948841.076065:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000905e873. 00000400:00000010:1.0:1784948841.076068:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ae8b415c. 00000400:00000010:1.0:1784948841.076071:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b666dac0. 00000400:00000010:1.0:1784948841.076073:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000017f0d388. 00000400:00000010:1.0:1784948841.076076:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000039ee8127. 00000400:00000010:1.0:1784948841.076079:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a400fa42. 00000400:00000010:1.0:1784948841.076081:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009b1df9a1. 00000400:00000010:1.0:1784948841.076084:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f99d1b9f. 00000400:00000010:1.0:1784948841.076086:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000534ace9d. 00000400:00000010:1.0:1784948841.076088:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000074570d70. 00000400:00000010:1.0:1784948841.076091:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007174c252. 00000400:00000010:1.0:1784948841.076093:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c0a3f621. 00000400:00000010:1.0:1784948841.076095:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b7bd3e11. 00000400:00000010:1.0:1784948841.076098:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000079f1d10e. 00000400:00000010:1.0:1784948841.076101:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000003ac0766. 00000400:00000010:1.0:1784948841.076103:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000077b9b581. 00000400:00000010:1.0:1784948841.076105:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000249338ff. 00000400:00000010:1.0:1784948841.076108:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000011546f5a. 00000400:00000010:1.0:1784948841.076110:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001e8db685. 00000400:00000010:1.0:1784948841.076112:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cdcf0a74. 00000400:00000010:1.0:1784948841.076115:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cd6cefc7. 00000400:00000010:1.0:1784948841.076117:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d8357cb9. 00000400:00000010:1.0:1784948841.076120:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000dda44ac8. 00000400:00000010:1.0:1784948841.076122:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009b22b71e. 00000400:00000010:1.0:1784948841.076124:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002d12d4bf. 00000400:00000010:1.0:1784948841.076127:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000061dbe3a3. 00000400:00000010:1.0:1784948841.076129:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d2a46502. 00000400:00000010:1.0:1784948841.076137:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000038aa1d2e. 00000400:00000010:1.0:1784948841.076144:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000e71a71c. 00000400:00000010:1.0:1784948841.076147:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003e3841d6. 00000400:00000010:1.0:1784948841.076150:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007b677d5c. 00000400:00000010:1.0:1784948841.076154:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000007e48c90. 00000400:00000010:1.0:1784948841.076157:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000279bae33. 00000400:00000010:1.0:1784948841.076159:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000094f3bbd4. 00000400:00000010:1.0:1784948841.076162:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e5d6f486. 00000400:00000010:1.0:1784948841.076164:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000acbf6818. 00000400:00000010:1.0:1784948841.076166:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000451e3fe4. 00000400:00000010:1.0:1784948841.076168:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009234a5fd. 00010000:00000001:2.0:1784948841.716073:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948841.716101:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948841.716111:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948841.716113:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948841.716209:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948841.716218:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948841.716223:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948841.908074:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948841.908100:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948841.908102:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948841.908111:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948841.908118:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948841.908123:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948841.908127:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948842.740048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948842.740057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948842.740065:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948842.740067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948842.740194:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948842.740201:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948842.740206:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948842.932005:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948842.932018:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948842.932019:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948842.932023:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948842.932025:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948842.932028:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948842.932030:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948843.764043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948843.764052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948843.764060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948843.764062:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948843.764119:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948843.764127:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948843.764131:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948843.956066:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948843.956083:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948843.956086:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948843.956093:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948843.956096:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948843.956101:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948843.956103:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948844.148067:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.9@tcp 00000800:00000010:0.0:1784948844.148078:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005a1c05a3. 00000400:00000200:0.0:1784948844.148091:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948844.148098:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948844.148107:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11117 health :900 00000400:00000200:0.0:1784948844.148115:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948844.148119:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:0.0:1784948844.148125:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000800:00000100:0.0:1784948844.148169:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000800:00000010:0.0:1784948844.148171:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c2b2d4f6. 00000400:00000200:0.0:1784948844.148177:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948844.148180:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948844.148184:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948844.148188:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:0.0:1784948844.148191:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:2.0:1784948844.148235:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948844.148243:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000001:2.0:1784948844.148278:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948844.148280:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948844.148283:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000010:2.0:1784948844.148285:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000400:00000200:2.0:1784948844.148289:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948844.148292:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:2.0:1784948844.148296:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000038ac69a2. 00000400:00000200:2.0:1784948844.148301:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:2.0:1784948844.148303:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000400:00000200:2.0:1784948844.148306:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000400:00000200:2.0:1784948844.148309:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:2.0:1784948844.148311:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948844.148311:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948844.148313:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000400:00000010:2.0:1784948844.148314:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000400:00000200:2.0:1784948844.148315:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948844.148316:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:2.0:1784948844.148317:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002c1a3d41. 00000400:00000200:2.0:1784948844.148320:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:2.0:1784948844.148322:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000400:00000200:2.0:1784948844.148323:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00010000:00000001:2.0:1784948844.788122:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948844.788136:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948844.788146:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948844.788147:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948844.788324:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948844.788332:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948844.788337:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948844.980097:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948844.980118:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948844.980122:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948844.980132:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948844.980137:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948844.980143:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948844.980146:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948845.172093:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948845.172105:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a2236612. 00000800:00000100:0.0:1784948845.172112:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000400:00000010:1.0:1784948845.172119:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a17860fa. 00000800:00000010:0.0:1784948845.172123:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a328590b. 00000400:00000010:1.0:1784948845.172128:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000010:1.0:1784948845.172141:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009234a5fd. 00000400:00000200:0.0:1784948845.172143:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948845.172153:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948845.172164:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948845.172180:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948845.172188:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948845.172192:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3290:3290:242:4294967295] 192.168.202.137@tcp LPNI seq info [2133:2133:8:4294967295] 00000400:00000200:1.0:1784948845.172200:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948845.172208:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948845.172214:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948845.172220:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000400:00000200:0.0:1784948845.172228:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11118 health :700 00000800:00000200:1.0:1784948845.172262:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784948845.172264:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948845.172294:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000800:00000200:1.0:1784948845.172303:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:0.0:1784948845.172314:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000800:00000200:1.0:1784948845.172317:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784948845.172383:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948845.172394:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:2.0:1784948845.172404:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948845.172408:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948845.172413:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:2.0:1784948845.172416:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000400:00000200:2.0:1784948845.172422:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948845.172427:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:2.0:1784948845.172432:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000be26f34f. 00000400:00000200:2.0:1784948845.172436:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:2.0:1784948845.172439:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:2.0:1784948845.172441:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000800:00000200:3.0:1784948845.172543:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948845.172552:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000800:00000001:2.0:1784948845.172554:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948845.172580:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948845.172587:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948845.172593:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948845.172593:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784948845.172601:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948845.172603:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784948845.172610:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948845.172614:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:2.0:1784948845.172628:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948845.172639:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948845.172652:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948845.172658:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3291:3291:243:4294967295] 192.168.202.137@tcp LPNI seq info [2134:2134:8:4294967295] 00000400:00000200:2.0:1784948845.172670:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948845.172676:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948845.172681:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000263f8d59. 00000800:00000200:2.0:1784948845.172687:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948845.172694:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948845.172697:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000263f8d59 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948845.172788:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948845.172797:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948845.172801:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000263f8d59. 00000400:00000200:2.0:1784948845.172804:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948845.172809:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948845.172816:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948845.172818:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948845.172839:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948845.172843:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948845.172848:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948845.172872:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948845.172893:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x84f9 00000800:00000001:0.0:1784948845.172901:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948845.172942:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948845.172946:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948845.172950:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948845.172957:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:0.0:1784948845.172960:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:0.0:1784948845.172964:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948845.172968:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a2236612. 00000400:00000010:0.0:1784948845.172972:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a17860fa. 00000800:00000001:0.0:1784948845.172981:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948845.172986:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948845.811987:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948845.811994:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948845.811999:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948845.812000:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948845.812054:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948845.812058:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948845.812061:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948846.004034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948846.004043:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948846.004058:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948820/real 1784948820] req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948845 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948846.004076:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1784948846.004081:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948846.004086:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784948846.004089:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000010:0.0:1784948846.004091:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d163afc3. 00000400:00000200:0.0:1784948846.004099:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:0.0:1784948846.004103:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00000010:1.0:1784948846.004103:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000002cc70ea. 00000100:00000001:0.0:1784948846.004115:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1784948846.004116:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000200:0.0:1784948846.004121:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948845 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948846.004130:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1784948846.004138:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948845 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948846.004139:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948846.004152:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.004153:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948846.004156:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948846.004162:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948846.004164:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948846.004167:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948845 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784948846.004169:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000001:0.0:1784948846.004181:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948846.004185:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.004191:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948846.004196:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948846.004199:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948846.004202:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948846.004207:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948846.004212:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948846.004223:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948845 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948846.004274:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784948846.004285:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948846.004305:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948846.004306:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948846.004310:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948846.004311:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.004313:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784948846.004318:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948846.004322:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1784948846.004324:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000002cc70ea] to pc [ptlrpcd_00_03+3] req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784948846.004339:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:0.0:1784948846.004343:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784948846.004347:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948845 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948846.004356:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289655168:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784948846.004361:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948846.004364:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948846.004367:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b16a92e x1871651289655168/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948845 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948846.004368:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948846.004375:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:0.0:1784948846.004382:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1784948846.004383:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948846.004389:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948846.004391:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948846.004398:0:335217:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1784948846.004399:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1784948846.004400:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784948846.004404:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000af0b2fc8. 00000100:00080000:1.0:1784948846.004406:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11092 00000100:00100000:3.0:1784948846.004409:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 02000000:00000001:0.0:1784948846.004411:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948846.004417:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784948846.004417:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.004418:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948846.004421:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000020:00000040:0.0:1784948846.004422:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:2.0:1784948846.004425:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948846.004429:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784948846.004430:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1784948846.004433:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11091 00000100:00000001:3.0:1784948846.004435:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784948846.004436:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000d8bbc1a8. 02000000:00000001:0.0:1784948846.004445:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:1.0:1784948846.004445:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:3.0:1784948846.004445:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948846.004450:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1784948846.004451:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1784948846.004455:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b16a92e. 00000100:00000001:3.0:1784948846.004457:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948846.004459:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948846.004462:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948846.004467:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948846.004467:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948846.004469:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948846.004473:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948846.004476:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948846.004481:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948846.004485:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948846.004485:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:0.0:1784948846.004492:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948846.004494:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948846.004496:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948846.004500:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:1.0:1784948846.004507:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948846.004513:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:3.0:1784948846.004520:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1784948846.004521:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:3.0:1784948846.004524:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948846.004527:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784948846.004530:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948846.004535:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948846.004536:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000001:3.0:1784948846.004540:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948846.004544:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1784948846.004546:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000002cc70ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289656448:0@lo:400: 00000100:00000040:1.0:1784948846.004551:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948846.004552:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784948846.004559:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:3.0:1784948846.004560:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948846.004565:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948846.004567:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:3.0:1784948846.004568:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948846.004576:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 00000100:00080000:1.0:1784948846.004577:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000001:3.0:1784948846.004583:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948846.004588:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000010:3.0:1784948846.004591:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000100:00000001:1.0:1784948846.004594:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948846.004600:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:3.0:1784948846.004600:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000100:00000200:3.0:1784948846.004608:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289656448, portal 25 02000000:00000001:1.0:1784948846.004611:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948846.004612:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948846.004615:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549163392 : -115544160388224 : ffff96e9c75d3b80) 02000000:00000001:1.0:1784948846.004620:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948846.004624:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948846.004627:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1784948846.004632:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.004637:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:1.0:1784948846.004642:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000926235a7. 00000100:00000040:3.0:1784948846.004642:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948846.004648:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000020:00000040:1.0:1784948846.004649:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:3.0:1784948846.004653:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289656448, offset 0 02000000:00000001:1.0:1784948846.004656:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1784948846.004661:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784948846.004662:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948846.004668:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948846.004670:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948846.004674:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948846.004679:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:1.0:1784948846.004684:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000049c99d00. 00000100:00000001:1.0:1784948846.004692:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948846.004693:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436c80 00000100:00100000:1.0:1784948846.004707:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1784948846.004707:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7b69 [2] + 0 00000400:00000010:3.0:1784948846.004713:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f13fb872. 00000400:00000200:3.0:1784948846.004721:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c5dc258d 00000100:00000001:1.0:1784948846.004731:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1784948846.004731:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948846.004735:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948846.004740:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:1.0:1784948846.004745:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000926235a7] to pc [ptlrpcd_rcv+-1] req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948846.004746:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c5dc258d 00000400:00000010:3.0:1784948846.004751:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000c5dc258d. 00000100:00000001:3.0:1784948846.004758:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948846.004762:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784948846.004767:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000078cfc26 x1871651289656448 msgsize 224 00000100:00100000:3.0:1784948846.004775:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784948846.004777:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948846.004785:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948846.004792:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948846.004795:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948846.004797:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948846.004804:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ddd830de. 00000400:00000200:3.0:1784948846.004805:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:3.0:1784948846.004810:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000020:00000040:1.0:1784948846.004812:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948846.004815:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948846.004820:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:3.0:1784948846.004821:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948846.004825:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948846.004830:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.004834:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1784948846.004838:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000040:3.0:1784948846.004839:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948846.004847:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.004849:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.004854:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784948846.004856:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784948846.004857:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948846.004860:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948846.004861:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1784948846.004862:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289656448 00000100:00000001:0.0:1784948846.004863:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948846.004868:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1784948846.004870:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948846.004871:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305785386 00000100:00000001:2.0:1784948846.004875:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:0.0:1784948846.004877:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.004881:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948846.004884:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 02000000:00000001:2.0:1784948846.004890:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.004896:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1784948846.004899:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289656448 00000100:00000001:0.0:1784948846.004930:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948846.004932:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948846.004935:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1784948846.004938:0:342757:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1784948846.004939:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948846.004941:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305785387 00000020:00000040:2.0:1784948846.004943:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784948846.004944:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1784948846.004945:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.004950:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784948846.004951:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948846.004952:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002b896c5a. 00000100:00100000:0.0:1784948846.004955:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000926235a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289656576:1.2.3.4@tcp:8: 00000020:00000040:2.0:1784948846.004955:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784948846.004963:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948846.004963:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784948846.004964:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784948846.004968:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ddd830de] to pc [ptlrpcd_00_00+0] req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948846.004969:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.004972:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948846.004972:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305785387 00000100:00000001:0.0:1784948846.004975:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948846.004981:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000100:00000001:2.0:1784948846.004982:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948846.004983:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.004990:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948846.004991:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948846.004991:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:3.0:1784948846.004998:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305785387 00000100:00000001:2.0:1784948846.005001:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948846.005003:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:3.0:1784948846.005004:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000029366a5. 00000020:00000010:2.0:1784948846.005007:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:0.0:1784948846.005008:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948846.005009:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948846.005017:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948846.005019:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948846.005020:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948846.005020:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948846.005020:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f1904ca. 00000100:00000001:3.0:1784948846.005026:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005027:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948846.005032:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11123) 00000020:00000010:2.0:1784948846.005032:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f7e68d33. 00000100:00000001:0.0:1784948846.005034:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.005035:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948846.005042:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000001:3.0:1784948846.005044:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948846.005049:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11123) 00000100:00000001:0.0:1784948846.005055:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948846.005057:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00100000:0.0:1784948846.005060:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:2.0:1784948846.005061:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948846.005065:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.005066:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948846.005069:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948846.005070:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948846.005074:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948846.005076:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005077:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948846.005082:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948846.005083:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005085:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948846.005091:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.005093:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005103:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948846.005106:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948846.005108:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948846.005111:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948846.005112:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948846.005113:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948846.005121:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289656704:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948846.005124:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948846.005127:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948846.005129:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.005129:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:0.0:1784948846.005132:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948846.005133:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000010:0.0:1784948846.005138:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001936ad9e. 00000100:00100000:2.0:1784948846.005143:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 717 02000000:00000001:0.0:1784948846.005144:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948846.005150:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000010:0.0:1784948846.005151:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d163afc3. 00000100:00000001:2.0:1784948846.005155:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782150736 : -115538927400880 : ffff96eaff464850) 00000400:00000010:0.0:1784948846.005159:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000100:00000040:2.0:1784948846.005163:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000078cfc26 x1871651289656448/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:507/0 lens 224/0 e 0 to 0 dl 1784948857 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948846.005166:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289656704, portal 10 00000100:00000001:0.0:1784948846.005173:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948846.005176:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549156224 : -115544160395392 : ffff96e9c75d1f80) 00000100:00000001:2.0:1784948846.005181:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948846.005186:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000040:0.0:1784948846.005187:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.005191:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948846.005198:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:0.0:1784948846.005206:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1784948846.005210:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000078cfc26 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289656448:12345-0@lo:400: 00000100:00000040:0.0:1784948846.005214:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:2.0:1784948846.005221:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289656448 00000400:00000010:0.0:1784948846.005222:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000020:00000001:2.0:1784948846.005225:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000200:0.0:1784948846.005227:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289656704, offset 0 00000020:00000001:2.0:1784948846.005249:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784948846.005254:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784948846.005255:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.005259:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948846.005277:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948846.005281:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:0.0:1784948846.005285:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:2.0:1784948846.005286:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948846.005294:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948846.005297:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:0.0:1784948846.005298:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1784948846.005300:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.005304:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:0.0:1784948846.005307:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:2.0:1784948846.005310:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:0.0:1784948846.005314:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1784948846.005316:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:0.0:1784948846.005320:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000010:2.0:1784948846.005324:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000936860e5. 00000400:00000200:0.0:1784948846.005327:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3292:3292:243:4294967295] 192.168.202.137@tcp LPNI seq info [2135:2135:8:4294967295] 02000000:00000001:2.0:1784948846.005331:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.005336:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.005339:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948846.005350:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00010000:00000040:2.0:1784948846.005351:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289656448 00000800:00000200:0.0:1784948846.005362:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00010000:00000001:2.0:1784948846.005362:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000010:0.0:1784948846.005372:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006487a147. 00010000:00000200:2.0:1784948846.005374:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000078cfc26 x1871651289656448/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:507/0 lens 224/224 e 0 to 0 dl 1784948857 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948846.005384:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948846.005399:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00010000:00000001:2.0:1784948846.005402:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000800:00000200:0.0:1784948846.005408:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000006487a147 type 1, nob 320 niov 1 nkiov 1 00010000:00000001:2.0:1784948846.005410:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948846.005420:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=18 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948846.005427:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948846.005432:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948846.005435:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.005436:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784948846.005445:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:0.0:1784948846.005446:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2062 usecs (3 jiffies) 02000000:00000001:2.0:1784948846.005448:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005455:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305785348 00000100:00000001:2.0:1784948846.005457:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948846.005460:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305785386 00000100:00000040:2.0:1784948846.005467:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948846.005469:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305785348 00000400:00000010:2.0:1784948846.005474:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000100:00000001:0.0:1784948846.005475:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305785387 00000100:00000001:0.0:1784948846.005483:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305785348 00000100:00000200:2.0:1784948846.005485:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289656448, offset 224 00000100:00000001:0.0:1784948846.005503:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305785387 00000100:00000001:0.0:1784948846.005510:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305785348 00000100:00000001:0.0:1784948846.005512:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305785387 00000400:00000200:2.0:1784948846.005516:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948846.005517:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948846.005522:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005525:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005528:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948846.005532:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784948846.005541:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436c80 00000400:00000200:2.0:1784948846.005546:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x84fd [1] + 224 00000400:00000200:2.0:1784948846.005551:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948846.005554:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948846.005558:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948846.005562:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948846.005570:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.005590:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784948846.005596:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:2.0:1784948846.005598:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000100:00000001:2.0:1784948846.005601:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948846.005603:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948846.005608:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948846.005612:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784948846.005615:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948846.005617:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.005619:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948846.005621:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.0:1784948846.005625:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948846.005626:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784948846.005626:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000078cfc26 x1871651289656448/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:507/0 lens 224/224 e 0 to 0 dl 1784948857 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784948846.005633:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000010:3.0:1784948846.005635:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006487a147. 00000400:00000001:0.0:1784948846.005637:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948846.005643:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000800:00000001:1.0:1784948846.005643:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948846.005645:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948846.005652:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000100:00100000:2.0:1784948846.005653:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000078cfc26 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289656448:12345-0@lo:400: Request processed in 466us (879us total) trans 0 rc 0/0 00000400:00000010:0.0:1784948846.005659:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000800:00000001:1.0:1784948846.005660:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948846.005660:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948846.005669:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:1.0:1784948846.005669:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948846.005673:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000800:00000001:1.0:1784948846.005676:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948846.005678:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 717 00000100:00000200:0.0:1784948846.005679:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948846.005679:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000040:2.0:1784948846.005687:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948846.005689:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1784948846.005691:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948846.005695:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948846.005701:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:3.0:1784948846.005702:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948846.005708:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f7e68d33. 00000400:00000200:1.0:1784948846.005715:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436d80 00000020:00000010:2.0:1784948846.005718:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000100:00000200:0.0:1784948846.005722:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948846.005727:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f1904ca. 00000400:00000200:1.0:1784948846.005733:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 156272 00000020:00000040:2.0:1784948846.005737:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948846.005742:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1784948846.005746:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000800:00000001:1.0:1784948846.005749:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000010:2.0:1784948846.005751:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000936860e5. 00000100:00000040:3.0:1784948846.005752:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948846.005755:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1784948846.005758:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000800:00000001:1.0:1784948846.005759:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948846.005760:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.005763:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948846.005769:0:337121:0:(client.c:1580:after_reply()) Process entered 00000400:00000200:1.0:1784948846.005769:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784948846.005772:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:0.0:1784948846.005775:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948846.005780:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784948846.005782:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:2.0:1784948846.005782:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bb87e206. 00000100:00000001:0.0:1784948846.005785:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948846.005790:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948846.005792:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948846.005792:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d3baa36a. 00000100:00000001:1.0:1784948846.005796:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948846.005796:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00001000:0.0:1784948846.005800:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=18 v=5 (5 5 5 5) 00000100:00000200:1.0:1784948846.005801:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948846.005809:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cf92476f. 00000100:00001000:0.0:1784948846.005810:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=18 v=5 (1 1 1 1) 00000100:00000040:1.0:1784948846.005816:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000cf92476f x1871651289656704 msgsize 224 00000100:00000001:0.0:1784948846.005817:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948846.005821:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948846.005826:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948846.005831:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948846.005831:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948846.005837:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948846.005842:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948846.005847:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948846.005849:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948846.005853:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948846.005855:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948846.005856:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784948846.005859:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948846.005862:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948846.005873:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000002cc70ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289656448:0@lo:400: 00000100:00000001:0.0:1784948846.005880:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948846.005882:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948846.005884:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000002cc70ea x1871651289656448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948846.005891:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948846.005893:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1784948846.005893:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000010:0.0:1784948846.005895:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 00000100:00100000:2.0:1784948846.005897:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289656704 02000000:00000001:0.0:1784948846.005900:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1784948846.005901:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948846.005927:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948846.005928:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948846.005931:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948846.005934:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000100:00100000:2.0:1784948846.005935:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289656704 00000020:00000040:0.0:1784948846.005939:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784948846.005940:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948846.005945:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948846.005946:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:2.0:1784948846.005948:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:0.0:1784948846.005950:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 00000020:00000040:2.0:1784948846.005952:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:0.0:1784948846.005956:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1784948846.005957:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:0.0:1784948846.005961:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:2.0:1784948846.005962:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000010:0.0:1784948846.005965:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000002cc70ea. 00000100:00000001:2.0:1784948846.005967:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948846.005971:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000010:2.0:1784948846.005973:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:0.0:1784948846.005974:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.005978:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948846.005979:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948846.005983:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:0.0:1784948846.005985:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784948846.005988:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000f1904ca. 00000100:00000001:0.0:1784948846.005989:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948846.005995:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1784948846.005995:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f7e68d33. 00000100:00000001:0.0:1784948846.005999:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948846.006002:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000040:2.0:1784948846.006010:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948846.006013:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948846.006014:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948846.006016:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.006020:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948846.006022:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784948846.006023:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.006034:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.006042:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948846.006044:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948846.006049:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 693 00000100:00000040:2.0:1784948846.006051:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948846.006053:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549149952 : -115544160401664 : ffff96e9c75d0700) 00000100:00000040:2.0:1784948846.006058:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000cf92476f x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:507/0 lens 224/0 e 0 to 0 dl 1784948857 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.006066:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948846.006068:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:2.0:1784948846.006070:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948846.006073:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00100000:2.0:1784948846.006076:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000cf92476f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289656704:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784948846.006079:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289656704 00000020:00000001:2.0:1784948846.006080:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948846.006083:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948846.006085:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.006086:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948846.006088:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784948846.006090:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948846.006092:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948846.006094:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948846.006095:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948846.006096:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.006098:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948846.006101:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948846.006102:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948846.006104:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000936860e5. 02000000:00000001:2.0:1784948846.006106:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.006107:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.006109:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948846.006111:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289656704 00010000:00000001:2.0:1784948846.006112:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948846.006115:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000cf92476f x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:507/0 lens 224/224 e 0 to 0 dl 1784948857 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948846.006121:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948846.006123:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948846.006124:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=13 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948846.006127:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948846.006129:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948846.006131:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948846.006133:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948846.006134:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.006135:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948846.006137:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948846.006139:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000100:00000200:2.0:1784948846.006141:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289656704, offset 224 00000400:00000200:2.0:1784948846.006145:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948846.006150:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948846.006155:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3293:3293:243:4294967295] 192.168.202.137@tcp LPNI seq info [2136:2136:8:4294967295] 00000400:00000200:2.0:1784948846.006163:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948846.006167:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948846.006170:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000263f8d59. 00000800:00000200:2.0:1784948846.006174:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948846.006178:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948846.006181:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000263f8d59 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948846.006192:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948846.006194:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948846.006196:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948846.006197:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948846.006199:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948846.006202:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cf92476f x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:507/0 lens 224/224 e 0 to 0 dl 1784948857 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948846.006210:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000cf92476f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289656704:12345-192.168.202.137@tcp:400: Request processed in 142us (393us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948846.006216:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 693 00000100:00000040:2.0:1784948846.006218:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948846.006220:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948846.006223:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948846.006225:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f7e68d33. 00000020:00000010:2.0:1784948846.006228:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948846.006251:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000f1904ca. 00000020:00000040:2.0:1784948846.006254:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948846.006256:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948846.006309:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948846.006313:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000263f8d59. 00000400:00000200:0.0:1784948846.006316:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948846.006321:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948846.006325:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:0.0:1784948846.006327:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000100:00000001:0.0:1784948846.006330:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948846.006331:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948846.006337:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948846.006342:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948846.006344:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948846.006346:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948846.006351:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948846.006359:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436d80 00000400:00000200:0.0:1784948846.006363:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8505 [1] + 224 00000800:00000001:0.0:1784948846.006368:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948846.006372:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948846.006374:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948846.006377:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948846.006379:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948846.006382:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948846.006389:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948846.006406:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948846.006410:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948846.006412:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.006441:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948846.006444:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948846.006448:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948846.006449:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948846.006451:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d163afc3. 00000400:00000200:2.0:1784948846.006455:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:2.0:1784948846.006457:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00000001:2.0:1784948846.006459:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948846.006463:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948846.006470:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.006476:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948846.006477:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.006479:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948846.006481:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948846.006482:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948846.006483:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948846.006484:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948846.006488:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=13 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948846.006491:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=13 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948846.006493:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948846.006494:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948846.006495:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948846.006498:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948846.006503:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.006506:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948846.006507:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948846.006508:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948846.006509:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948846.006512:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.006517:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948846.006519:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948846.006520:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948846.006522:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948846.006528:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289656704:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948846.006532:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.006533:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948846.006535:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ddd830de x1871651289656704/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948862 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948846.006540:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948846.006541:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948846.006543:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001936ad9e. 02000000:00000001:2.0:1784948846.006545:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948846.006546:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948846.006547:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948846.006548:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948846.006549:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 02000000:00000001:2.0:1784948846.006552:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948846.006553:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948846.006554:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ddd830de. 00000100:00000001:2.0:1784948846.006557:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948846.006558:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.006559:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948846.006562:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948846.006563:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948846.006565:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948846.006566:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948846.047019:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305785386, at 4305785387 00000100:00000010:0.0:1784948846.047034:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000040:3.0:1784948846.048032:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305785387, at 4305785388 00000100:00000010:3.0:1784948846.048045:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000029366a5. 00000100:00000040:2.0:1784948846.048049:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305785387, at 4305785388 00000100:00000040:1.0:1784948846.048054:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305785387, at 4305785388 00000100:00000010:2.0:1784948846.048067:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000100:00000010:1.0:1784948846.048072:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002b896c5a. 00010000:00000001:2.0:1784948846.835994:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948846.836001:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948846.836006:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948846.836007:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948846.836057:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948846.836062:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948846.836064:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948847.028032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948847.028055:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948847.028071:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948846/real 0] req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948847 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.028088:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.028090:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948847.028092:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948847.028095:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948847 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.028103:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.028112:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948847.028114:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.028117:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948847.028120:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948847.028122:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948847.028125:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948847 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.028132:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784948847.028152:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.028154:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948847.028155:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.028158:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784948847.028160:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948847.028164:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784948847.028192:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784948847.028198:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784948847.028204:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948847 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948847.028216:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000926235a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289656576:1.2.3.4@tcp:8: 00000100:00000040:1.0:1784948847.028293:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948847.028296:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948847.028300:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948847.028301:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948847.028306:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000926235a7 x1871651289656576/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948847 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784948847.028310:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000299dc118. 00000100:00000001:0.0:1784948847.028316:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000040:1.0:1784948847.028317:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948847.028320:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948847.028322:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:1.0:1784948847.028322:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1784948847.028325:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:1.0:1784948847.028325:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:0.0:1784948847.028330:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 0000000049c99d00. 02000000:00000001:1.0:1784948847.028331:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948847.028340:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:1.0:1784948847.028340:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e79c6895. 02000000:00000001:0.0:1784948847.028342:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948847.028344:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000926235a7. 00000100:00000001:1.0:1784948847.028348:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.028349:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948847.028351:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.028353:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948847.028354:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948847.028358:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948847.028360:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.028363:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948847.028365:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.028368:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948847.028371:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948847.028377:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000299dc118] to pc [ptlrpcd_00_01+1] req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948847.028422:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948847.028425:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948847.028428:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948847.028431:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948847.028432:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948847.028436:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784948847.028440:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11092 00000400:00000200:1.0:1784948847.028450:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:1.0:1784948847.028454:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11118 00000400:00000200:1.0:1784948847.028459:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:1.0:1784948847.028462:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1784948847.028466:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948847.028469:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948847.028474:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948847.028476:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:1.0:1784948847.028480:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948847.028480:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948847.028483:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:2.0:1784948847.028484:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948847.028488:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:3.0:1784948847.028491:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1784948847.028492:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948847.028493:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:3.0:1784948847.028501:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948847.028502:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:1.0:1784948847.028511:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:1.0:1784948847.028515:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948847.028516:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948847.028518:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784948847.028521:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948847.028523:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948847.028530:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784948847.028531:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948847.028533:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:3.0:1784948847.028536:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1784948847.028541:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000001:3.0:1784948847.028543:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1784948847.028545:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:1.0:1784948847.028560:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948847.028578:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1784948847.028585:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948847.028587:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.028590:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1784948847.028592:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948847.028596:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784948847.028598:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb4f6eba. 00000020:00000040:1.0:1784948847.028604:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:1.0:1784948847.028611:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1784948847.028615:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.028620:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948847.028625:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948847.028630:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.028631:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:1.0:1784948847.028637:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000009e9a21e3. 00000100:00000001:3.0:1784948847.028638:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.028645:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948847.028649:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1784948847.028655:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1784948847.028666:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948847.028676:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948847.028688:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289656832:0@lo:400: 00000100:00000001:1.0:1784948847.028690:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948847.028694:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948847.028702:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb4f6eba] to pc [ptlrpcd_rcv+-1] req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948847.028712:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948847.028724:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948847.028728:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948847.028733:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948847.028734:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948847.028740:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:3.0:1784948847.028742:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 00000100:00000001:1.0:1784948847.028751:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:3.0:1784948847.028751:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948847.028759:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007bd58f9d. 00000400:00000010:3.0:1784948847.028764:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f13fb872. 00000020:00000040:1.0:1784948847.028766:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948847.028775:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948847.028777:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1784948847.028778:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 02000000:00000001:1.0:1784948847.028781:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948847.028791:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289656832, portal 25 02000000:00000010:1.0:1784948847.028792:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000adf08f9b. 00000100:00000001:0.0:1784948847.028794:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948847.028798:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948847.028800:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.028801:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948847.028802:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948847.028808:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549152640 : -115544160398976 : ffff96e9c75d1180) 00000100:00000040:1.0:1784948847.028809:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784948847.028812:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948847.028825:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.028835:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948847.028835:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948847.028841:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.028841:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948847.028850:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948847.028852:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948847.028855:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bd58f9d] to pc [ptlrpcd_00_02+2] req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948847.028856:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948847.028864:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948847.028864:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00100000:0.0:1784948847.028872:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289656960:192.168.202.137@tcp:8: 00000400:00000010:3.0:1784948847.028875:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000001:0.0:1784948847.028881:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000200:3.0:1784948847.028884:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289656832, offset 0 02000000:00000001:0.0:1784948847.028888:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948847.028892:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.028896:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948847.028897:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948847.028902:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:0.0:1784948847.028938:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000af0b2fc8. 00000100:00000040:1.0:1784948847.028939:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11124) 02000000:00000001:0.0:1784948847.028944:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948847.028951:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ebd65351. 00000400:00000010:0.0:1784948847.028982:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 00000100:00000001:1.0:1784948847.028988:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784948847.028990:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:0.0:1784948847.028992:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289656960, portal 4 00000100:00000001:1.0:1784948847.028993:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948847.028999:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948847.028999:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948847.029002:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549148160 : -115544160403456 : ffff96e9c75d0000) 00000100:00000040:1.0:1784948847.029006:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948847.029015:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948847.029016:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436e00 00000100:00000001:1.0:1784948847.029032:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948847.029033:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948847.029035:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948847.029040:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7b89 [2] + 0 00000100:00000040:0.0:1784948847.029041:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784948847.029042:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948847.029047:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948847.029050:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000089c8262f. 00000400:00000010:3.0:1784948847.029054:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fde3a35c. 02000000:00000001:1.0:1784948847.029055:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948847.029065:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289657088:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784948847.029069:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000096ea68f5 00000100:00000001:1.0:1784948847.029073:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000200:0.0:1784948847.029077:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289656960, offset 0 02000000:00000001:1.0:1784948847.029077:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948847.029080:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948847.029083:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948847.029085:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948847.029087:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000010:1.0:1784948847.029088:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dbce179e. 02000000:00000001:1.0:1784948847.029109:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948847.029116:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000010:1.0:1784948847.029119:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000768f23df. 00000400:00000200:0.0:1784948847.029122:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000010:1.0:1784948847.029129:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000200:0.0:1784948847.029130:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:1.0:1784948847.029133:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657088, portal 10 00000400:00000200:0.0:1784948847.029136:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784948847.029136:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948847.029139:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549161600 : -115544160390016 : ffff96e9c75d3480) 00000400:00000200:0.0:1784948847.029142:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1784948847.029146:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948847.029147:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948847.029153:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3294:3294:243:4294967295] 192.168.202.137@tcp LPNI seq info [2137:2137:8:4294967295] 00000100:00000001:1.0:1784948847.029159:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948847.029162:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948847.029167:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784948847.029168:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000400:00000200:0.0:1784948847.029174:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000200:1.0:1784948847.029174:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289657088, offset 0 00000100:00000001:2.0:1784948847.029178:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948847.029187:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000800:00000200:0.0:1784948847.029188:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948847.029195:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096ea68f5 00000800:00000010:0.0:1784948847.029196:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000086e4d21. 00000400:00000200:1.0:1784948847.029200:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000010:3.0:1784948847.029205:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000096ea68f5. 00000400:00000200:1.0:1784948847.029209:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:0.0:1784948847.029213:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784948847.029218:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948847.029220:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:1.0:1784948847.029272:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000200:3.0:1784948847.029273:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1784948847.029280:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:1.0:1784948847.029280:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:3.0:1784948847.029284:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000d9986ea8 x1871651289656832 msgsize 224 00000800:00000200:0.0:1784948847.029287:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000086e4d21 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1784948847.029288:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3295:3295:242:4294967295] 192.168.202.137@tcp LPNI seq info [2138:2138:7:4294967295] 00000100:00100000:3.0:1784948847.029304:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784948847.029311:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784948847.029312:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948847.029313:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948847.029321:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000800:00000200:1.0:1784948847.029322:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948847.029326:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305786411 00000800:00000010:1.0:1784948847.029326:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000100:00000001:3.0:1784948847.029330:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000010:0.0:1784948847.029333:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000800:00000200:1.0:1784948847.029335:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948847.029343:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 0 00000800:00000200:1.0:1784948847.029343:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000200:3.0:1784948847.029345:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000800:00000200:1.0:1784948847.029348:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948847.029352:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305786411 00000400:00000010:3.0:1784948847.029353:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000010:0.0:1784948847.029359:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:3.0:1784948847.029363:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948847.029367:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.029370:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 1 00000100:00000040:1.0:1784948847.029373:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2062 usecs (3 jiffies) 00000100:00000200:3.0:1784948847.029375:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.029377:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305786411 00000100:00000001:1.0:1784948847.029378:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305786372 00000100:00000001:1.0:1784948847.029386:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305786411 00000100:00000010:0.0:1784948847.029388:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:1.0:1784948847.029393:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305786372 00000100:00000001:0.0:1784948847.029396:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784948847.029399:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305786411 00000100:00000001:0.0:1784948847.029402:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305786411 00000100:00000001:1.0:1784948847.029406:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305786372 00000100:00000010:0.0:1784948847.029410:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:1.0:1784948847.029413:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305786411 00000100:00000001:0.0:1784948847.029415:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948847.029416:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305786372 00000100:00000001:3.0:1784948847.029416:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948847.029421:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948847.029423:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305786411 00000100:00000001:0.0:1784948847.029425:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948847.029427:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948847.029428:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948847.029429:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.029435:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.029436:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948847.029442:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.029444:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948847.029447:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.029449:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1784948847.029455:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948847.029456:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948847.029459:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948847.029459:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948847.029462:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.029464:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.029465:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948847.029468:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948847.029471:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.029471:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.029471:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948847.029479:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.029493:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.029501:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305786411 00000100:00000001:3.0:1784948847.029506:0:337119:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, keep current deadline 4305786411 00000100:00000001:1.0:1784948847.029508:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948847.029511:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305786411 00000100:00100000:1.0:1784948847.029514:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289656832 00000100:00000001:3.0:1784948847.029517:0:337119:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, keep current deadline 4305786411 02000000:00000001:1.0:1784948847.029519:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948847.029525:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948847.029526:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305786411 00000100:00000001:1.0:1784948847.029531:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.029534:0:337119:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, keep current deadline 4305786411 02000000:00000001:1.0:1784948847.029539:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.029541:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305786411 00000100:00100000:1.0:1784948847.029548:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289656832 00000100:00000001:3.0:1784948847.029550:0:337119:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, keep current deadline 4305786411 00000020:00000001:1.0:1784948847.029555:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948847.029558:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784948847.029560:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784948847.029565:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948847.029567:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:2.0:1784948847.029571:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000040:1.0:1784948847.029574:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948847.029574:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784948847.029578:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000086e4d21. 00000020:00000001:1.0:1784948847.029584:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948847.029585:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948847.029587:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948847.029591:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948847.029594:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948847.029601:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784948847.029602:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948847.029607:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948847.029609:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007d5987e9. 00000400:00000200:2.0:1784948847.029610:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000800:00000001:0.0:1784948847.029615:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948847.029615:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784948847.029616:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000020:00000010:1.0:1784948847.029625:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007d5987e9. 00000100:00000001:2.0:1784948847.029626:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948847.029634:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948847.029638:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bf7eadde. 00000800:00000001:0.0:1784948847.029640:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948847.029648:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 00000100:00000001:2.0:1784948847.029654:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784948847.029656:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000010:1.0:1784948847.029668:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:2.0:1784948847.029669:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948872 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948847.029684:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948847.029685:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948847.029686:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6436e80 00000100:00000001:1.0:1784948847.029691:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948847.029693:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948847.029695:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000100:0.0:1784948847.029699:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289656960 offset 0 length 520: 4 00000100:00000001:1.0:1784948847.029700:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948847.029705:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000fde54599 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:0.0:1784948847.029709:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:1.0:1784948847.029710:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948847.029721:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948847.029722:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948847.029739:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948847.029742:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948847.029750:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 718 00000100:00000040:1.0:1784948847.029756:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948847.029760:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782164048 : -115538927387568 : ffff96eaff467c50) 00000100:00000040:1.0:1784948847.029768:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d9986ea8 x1871651289656832/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:508/0 lens 224/0 e 0 to 0 dl 1784948858 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948847.029773:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784948847.029780:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948847.029785:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948847.029787:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000010:3.0:1784948847.029788:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000800:00000001:2.0:1784948847.029789:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:1.0:1784948847.029796:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007d5987e9. 00000800:00000001:2.0:1784948847.029796:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948847.029799:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948847.029805:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784948847.029814:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000010:1.0:1784948847.029816:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007d5987e9. 00000400:00000200:3.0:1784948847.029819:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:1.0:1784948847.029830:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d9986ea8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289656832:12345-0@lo:400: 00000400:00000200:3.0:1784948847.029830:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000800:00000001:0.0:1784948847.029833:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948847.029835:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6436f00 00000400:00000010:3.0:1784948847.029837:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000800:00000001:0.0:1784948847.029840:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948847.029849:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948847.029849:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289656832 00000100:00000001:3.0:1784948847.029852:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784948847.029855:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:2.0:1784948847.029859:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 156496 00000020:00000001:1.0:1784948847.029863:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:3.0:1784948847.029865:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948847.029871:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948847.029874:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948847.029881:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948847.029887:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784948847.029890:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:2.0:1784948847.029892:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948847.029896:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:2.0:1784948847.029898:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784948847.029900:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948847.029902:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948847.029942:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784948847.029944:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784948847.029949:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:1.0:1784948847.029951:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:2.0:1784948847.029954:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:1.0:1784948847.029957:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948847.029975:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000010:2.0:1784948847.029976:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fdc32c74. 00000100:00000001:3.0:1784948847.029977:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.029982:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:2.0:1784948847.029985:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000fdc32c74 x1871651289657088 msgsize 224 00000100:00000001:3.0:1784948847.029986:0:337113:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1784948847.029988:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:2.0:1784948847.029996:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000010:1.0:1784948847.029999:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000004cb3d562. 02000000:00000001:1.0:1784948847.030005:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.030009:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948847.030009:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:1.0:1784948847.030016:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948847.030019:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000040:1.0:1784948847.030021:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289656832 00000800:00000001:2.0:1784948847.030026:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948847.030028:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948847.030038:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d9986ea8 x1871651289656832/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:508/0 lens 224/224 e 0 to 0 dl 1784948858 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948847.030051:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948847.030053:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.030055:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948847.030058:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657088 00000100:00001000:1.0:1784948847.030058:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=19 v=5 (1 1 1 1) 02000000:00000001:0.0:1784948847.030060:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948847.030078:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948847.030079:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948847.030082:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948847.030082:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 02000000:00000001:0.0:1784948847.030086:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.030086:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948847.030090:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00100000:0.0:1784948847.030091:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657088 00000020:00000001:0.0:1784948847.030093:0:343544:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784948847.030093:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948847.030096:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:1.0:1784948847.030098:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1784948847.030099:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1784948847.030101:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000040:0.0:1784948847.030102:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948847.030106:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000010:1.0:1784948847.030106:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000020:00000001:0.0:1784948847.030110:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784948847.030112:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289656832, offset 224 00000100:00000001:0.0:1784948847.030115:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948847.030119:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000400:00000200:1.0:1784948847.030119:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948847.030123:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948847.030126:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000400:00000200:1.0:1784948847.030130:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784948847.030132:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000069d2b91d. 00000400:00000200:1.0:1784948847.030138:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436e00 00000020:00000010:0.0:1784948847.030139:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000356850de. 00000020:00000010:0.0:1784948847.030144:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:1.0:1784948847.030146:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8519 [1] + 224 00000100:00000040:0.0:1784948847.030154:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:1.0:1784948847.030155:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948847.030157:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948847.030160:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:1.0:1784948847.030160:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948847.030163:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948847.030166:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948847.030170:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:1.0:1784948847.030172:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.030173:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948847.030175:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948847.030187:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.030194:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.030208:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948847.030210:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948847.030212:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:0.0:1784948847.030217:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 694 00000400:00000200:1.0:1784948847.030219:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000100:00000040:0.0:1784948847.030247:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000010:1.0:1784948847.030247:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000100:00000001:0.0:1784948847.030251:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470024192 : -115544239527424 : ffff96e9c2a5aa00) 00000100:00000001:1.0:1784948847.030270:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948847.030283:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000040:0.0:1784948847.030288:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000fdc32c74 x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:508/0 lens 224/0 e 0 to 0 dl 1784948858 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948847.030290:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948847.030296:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784948847.030300:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784948847.030301:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:1.0:1784948847.030303:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948847.030306:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000020:00000001:1.0:1784948847.030307:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784948847.030307:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948847.030310:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948847.030314:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:0.0:1784948847.030315:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000100:00000040:1.0:1784948847.030315:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d9986ea8 x1871651289656832/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:508/0 lens 224/224 e 0 to 0 dl 1784948858 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948847.030325:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000fdc32c74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289657088:12345-192.168.202.137@tcp:400: 00000400:00000001:3.0:1784948847.030326:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:1.0:1784948847.030332:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d9986ea8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289656832:12345-0@lo:400: Request processed in 544us (1046us total) trans 0 rc 0/0 00000400:00000001:3.0:1784948847.030332:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:0.0:1784948847.030333:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657088 00000020:00000001:0.0:1784948847.030338:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:3.0:1784948847.030339:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f13fb872. 00000020:00000001:0.0:1784948847.030343:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00100000:1.0:1784948847.030347:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 718 00000020:00000001:0.0:1784948847.030348:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948847.030350:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000020:00000001:0.0:1784948847.030352:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:1.0:1784948847.030352:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000010:3.0:1784948847.030356:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000020:00000001:0.0:1784948847.030358:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784948847.030358:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948847.030361:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:0.0:1784948847.030365:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784948847.030366:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1784948847.030369:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948847.030369:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000020:00000001:0.0:1784948847.030374:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:3.0:1784948847.030376:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948847.030377:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000001:0.0:1784948847.030378:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948847.030382:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948847.030385:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bf7eadde. 00000020:00000001:0.0:1784948847.030388:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000040:1.0:1784948847.030392:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948847.030393:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948847.030397:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:1.0:1784948847.030400:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784948847.030403:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b9f7c212. 02000000:00000010:1.0:1784948847.030405:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000004cb3d562. 02000000:00000001:0.0:1784948847.030410:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948847.030412:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948847.030414:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948847.030416:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948847.030418:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948847.030418:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:0.0:1784948847.030426:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657088 00000100:00000200:1.0:1784948847.030428:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:0.0:1784948847.030433:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000010:1.0:1784948847.030436:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000069afdf3e. 00010000:00000200:0.0:1784948847.030443:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000fdc32c74 x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:508/0 lens 224/224 e 0 to 0 dl 1784948858 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948847.030443:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b3c5fd98. 00000100:00000001:3.0:1784948847.030449:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948847.030456:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948847.030460:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948847.030463:0:337119:0:(client.c:1580:after_reply()) Process entered 00010000:00000001:0.0:1784948847.030466:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948847.030470:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00001000:0.0:1784948847.030472:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=14 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948847.030475:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948847.030481:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.030483:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784948847.030488:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948847.030501:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948847.030508:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00001000:3.0:1784948847.030508:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=19 v=5 (5 5 5 5) 02000000:00000001:0.0:1784948847.030516:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00001000:3.0:1784948847.030520:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=19 v=5 (1 1 1 1) 02000000:00000001:0.0:1784948847.030521:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.030528:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948847.030531:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000040:0.0:1784948847.030535:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948847.030537:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948847.030540:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00010000:00000001:3.0:1784948847.030545:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:0.0:1784948847.030546:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289657088, offset 224 00010000:00000001:3.0:1784948847.030552:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948847.030556:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948847.030561:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948847.030567:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000040:3.0:1784948847.030588:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948847.030596:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3296:3296:243:4294967295] 192.168.202.137@tcp LPNI seq info [2139:2139:8:4294967295] 00000400:00000200:0.0:1784948847.030624:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948847.030624:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948847.030633:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000200:0.0:1784948847.030639:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948847.030639:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948847.030663:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d1b12746. 00000100:00000040:3.0:1784948847.030669:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948847.030675:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948847.030690:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948847.030698:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d1b12746 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:3.0:1784948847.030711:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289656832:0@lo:400: 00000100:00000001:3.0:1784948847.030720:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948847.030725:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948847.030726:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948847.030731:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000299dc118 x1871651289656832/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948847.030739:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948847.030749:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948847.030754:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948847.030758:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784948847.030761:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 02000000:00000001:3.0:1784948847.030765:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948847.030770:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 00000100:00000040:0.0:1784948847.030774:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fdc32c74 x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:508/0 lens 224/224 e 0 to 0 dl 1784948858 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948847.030779:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948847.030785:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948847.030790:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784948847.030800:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948847.030805:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e79c6895. 00000100:00100000:0.0:1784948847.030812:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000fdc32c74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289657088:12345-192.168.202.137@tcp:400: Request processed in 504us (819us total) trans 0 rc 0/0 02000000:00000001:3.0:1784948847.030815:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948847.030820:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948847.030826:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000299dc118. 00000100:00000001:3.0:1784948847.030835:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00100000:0.0:1784948847.030836:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 694 00000100:00000001:3.0:1784948847.030842:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948847.030845:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948847.030848:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948847.030851:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948847.030852:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000010:1.0:1784948847.030856:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d1b12746. 00000100:00000001:0.0:1784948847.030857:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948847.030859:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784948847.030862:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000356850de. 00000400:00000200:1.0:1784948847.030863:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948847.030866:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948847.030870:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000400:00000200:1.0:1784948847.030874:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948847.030876:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784948847.030878:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000069d2b91d. 00000400:00000200:1.0:1784948847.030881:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdaa2431 00000100:00000001:3.0:1784948847.030883:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948847.030885:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdaa2431. 00000020:00000040:0.0:1784948847.030887:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948847.030893:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948847.030893:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948847.030896:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784948847.030941:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948847.030951:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948847.030955:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948847.030960:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948847.030976:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948847.030989:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6436f00 00000400:00000200:1.0:1784948847.030997:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8529 [1] + 224 00000800:00000001:1.0:1784948847.031005:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948847.031011:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948847.031014:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948847.031020:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948847.031025:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948847.031031:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948847.031046:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948847.031070:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948847.031078:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948847.031083:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.031114:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948847.031119:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948847.031124:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948847.031127:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948847.031129:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000768f23df. 00000400:00000200:0.0:1784948847.031136:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:0.0:1784948847.031139:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000100:00000001:0.0:1784948847.031142:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948847.031147:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948847.031155:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.031163:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948847.031164:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.031166:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948847.031168:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948847.031170:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948847.031171:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948847.031173:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948847.031178:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=14 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948847.031181:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=14 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948847.031184:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948847.031185:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948847.031187:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948847.031190:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948847.031198:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.031201:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948847.031203:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948847.031205:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948847.031206:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948847.031209:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.031261:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948847.031274:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948847.031275:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948847.031279:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948847.031286:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289657088:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948847.031289:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.031291:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948847.031293:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007bd58f9d x1871651289657088/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948847.031299:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948847.031301:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948847.031302:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dbce179e. 02000000:00000001:0.0:1784948847.031308:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948847.031310:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948847.031311:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948847.031313:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948847.031314:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000adf08f9b. 02000000:00000001:0.0:1784948847.031317:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948847.031318:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948847.031319:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007bd58f9d. 00000100:00000001:0.0:1784948847.031322:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948847.031323:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.031325:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948847.031328:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948847.031330:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948847.031331:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948847.031333:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948847.071988:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305786411, at 4305786412 00000100:00000040:3.0:1784948847.072000:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305786411, at 4305786412 00000100:00000010:1.0:1784948847.072005:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000010:3.0:1784948847.072012:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:2.0:1784948847.072029:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305786411, at 4305786412 00000100:00000040:0.0:1784948847.072030:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305786411, at 4305786412 00000100:00000010:0.0:1784948847.072044:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:2.0:1784948847.072046:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00010000:00000001:2.0:1784948847.860005:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948847.860011:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948847.860017:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948847.860018:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948847.860081:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948847.860087:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948847.860092:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1784948848.244054:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000800:00000010:0.0:1784948848.244065:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003a8a98bc. 00000400:00000200:0.0:1784948848.244080:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948848.244090:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948848.244101:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11121 health :900 00000400:00000200:0.0:1784948848.244110:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948848.244116:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:0.0:1784948848.244123:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000800:00000100:0.0:1784948848.244165:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.8@tcp 00000800:00000010:0.0:1784948848.244170:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009ff40a64. 00000400:00000200:0.0:1784948848.244177:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948848.244187:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948848.244189:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948848.244194:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948848.244204:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000200:0.0:1784948848.244205:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:0.0:1784948848.244236:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000001:2.0:1784948848.244264:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948848.244269:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948848.244274:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000400:00000010:2.0:1784948848.244279:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000400:00000200:2.0:1784948848.244286:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948848.244293:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:2.0:1784948848.244299:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000053ab16eb. 00000400:00000200:2.0:1784948848.244307:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:2.0:1784948848.244310:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:2.0:1784948848.244314:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:2.0:1784948848.244318:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:2.0:1784948848.244321:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948848.244322:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948848.244324:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0c9b918 00000400:00000010:2.0:1784948848.244326:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f0c9b918. 00000400:00000200:2.0:1784948848.244328:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948848.244330:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:2.0:1784948848.244333:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b105ef1a. 00000400:00000200:2.0:1784948848.244336:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-110 00000400:00000200:2.0:1784948848.244338:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -110 00000400:00000200:2.0:1784948848.244340:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00010000:00000001:2.0:1784948848.884041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948848.884051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948848.884058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948848.884060:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948848.884149:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948848.884155:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948848.884160:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:1.0:1784948849.268057:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:0.0:1784948849.268061:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000800:00000010:0.0:1784948849.268072:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000057c7ab4. 00000400:00000010:1.0:1784948849.268075:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000057de27a4. 00000400:00000200:0.0:1784948849.268086:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948849.268098:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784948849.268100:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f7a7941e. 00000400:00000200:0.0:1784948849.268112:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11122 health :700 00000400:00000010:1.0:1784948849.268114:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000400:00000200:0.0:1784948849.268126:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:1.0:1784948849.268134:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000451e3fe4. 00000400:00000200:0.0:1784948849.268137:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:1.0:1784948849.268151:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948849.268155:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:1.0:1784948849.268234:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948849.268246:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948849.268255:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3297:3297:245:4294967295] 192.168.202.137@tcp LPNI seq info [2140:2140:8:4294967295] 00000400:00000200:1.0:1784948849.268268:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948849.268279:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948849.268288:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948849.268298:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000400:00000200:2.0:1784948849.268302:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:1.0:1784948849.268314:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784948849.268321:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000001:2.0:1784948849.268334:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1784948849.268336:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:2.0:1784948849.268342:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1784948849.268348:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784948849.268351:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:2.0:1784948849.268360:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000400:00000200:2.0:1784948849.268368:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948849.268378:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:2.0:1784948849.268384:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000400:00000200:2.0:1784948849.268393:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:2.0:1784948849.268400:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:2.0:1784948849.268408:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000800:00000200:3.0:1784948849.268620:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948849.268628:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784948849.268628:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:3.0:1784948849.268640:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948849.268647:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784948849.268652:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948849.268654:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784948849.268663:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784948849.268665:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948849.268682:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948849.268697:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948849.268718:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948849.268732:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948849.268748:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948849.268757:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3298:3298:246:4294967295] 192.168.202.137@tcp LPNI seq info [2141:2141:8:4294967295] 00000400:00000200:2.0:1784948849.268775:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948849.268784:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948849.268791:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009bb74647. 00000800:00000200:2.0:1784948849.268800:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948849.268809:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948849.268815:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009bb74647 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784948849.268969:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948849.268972:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948849.268974:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784948849.268981:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784948849.268985:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009bb74647. 00000800:00000001:3.0:1784948849.268987:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948849.268995:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948849.268998:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948849.269001:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948849.269021:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948849.269022:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948849.269053:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x853d 00000800:00000001:3.0:1784948849.269063:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948849.269072:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948849.269078:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948849.269085:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948849.269093:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:3.0:1784948849.269097:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000400:00000200:3.0:1784948849.269103:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948849.269107:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000057de27a4. 00000400:00000010:3.0:1784948849.269113:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f7a7941e. 00000800:00000001:3.0:1784948849.269126:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948849.269131:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948849.908016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948849.908024:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948849.908031:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948849.908033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948849.908103:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948849.908109:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948849.908113:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1784948850.292027:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:0.0:1784948850.292033:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003ed79484. 00000400:00000200:0.0:1784948850.292043:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948850.292048:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948850.292054:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11123 health :900 00000400:00000200:0.0:1784948850.292059:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948850.292063:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:0.0:1784948850.292067:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:2.0:1784948850.292133:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948850.292142:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:2.0:1784948850.292147:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948850.292150:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948850.292153:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000400:00000010:2.0:1784948850.292155:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000400:00000200:2.0:1784948850.292159:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948850.292163:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:2.0:1784948850.292166:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000043c3e1d8. 00000400:00000200:2.0:1784948850.292171:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000400:00000200:2.0:1784948850.292173:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000400:00000200:2.0:1784948850.292176:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00010000:00000001:2.0:1784948850.932048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948850.932058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948850.932066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948850.932071:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948850.932074:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948850.932078:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948850.932081:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948850.932084:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948850.932087:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948850.932089:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948850.932265:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948850.932271:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948850.932276:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1784948851.316016:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000800:00000010:0.0:1784948851.316026:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000036e798e7. 00000400:00000200:0.0:1784948851.316039:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948851.316050:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948851.316063:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948851.316071:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:1.0:1784948851.316085:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:0.0:1784948851.316087:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:1.0:1784948851.316106:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000090a03b90. 00000400:00000010:1.0:1784948851.316136:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000378c7327. 00000400:00000010:1.0:1784948851.316148:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000010:1.0:1784948851.316162:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000acbf6818. 00000400:00000200:2.0:1784948851.316175:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784948851.316176:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948851.316244:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948851.316255:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948851.316262:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3299:3299:248:4294967295] 192.168.202.137@tcp LPNI seq info [2142:2142:8:4294967295] 00000400:00000200:1.0:1784948851.316274:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948851.316287:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948851.316300:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948851.316305:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000800:00000010:1.0:1784948851.316312:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000400:00000001:2.0:1784948851.316318:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948851.316325:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948851.316331:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000800:00000200:1.0:1784948851.316332:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:2.0:1784948851.316339:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000400:00000200:2.0:1784948851.316350:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000200:1.0:1784948851.316357:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:2.0:1784948851.316365:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000800:00000200:1.0:1784948851.316371:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 2, nob 96 niov 1 nkiov 0 00000400:00000010:2.0:1784948851.316375:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009802f36f. 00000400:00000200:2.0:1784948851.316391:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:2.0:1784948851.316397:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:2.0:1784948851.316405:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000400:00000200:1.0:1784948851.316420:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11125 health :800 00000800:00000200:0.0:1784948851.316567:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948851.316572:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000400:00000200:0.0:1784948851.316579:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948851.316587:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948851.316594:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948851.316598:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784948851.316611:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948851.316624:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948851.316628:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948851.316635:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948851.316642:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948851.316655:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948851.316665:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948851.316686:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948851.316692:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3300:3300:248:4294967295] 192.168.202.137@tcp LPNI seq info [2143:2143:8:4294967295] 00000400:00000200:0.0:1784948851.316703:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948851.316709:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948851.316713:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b84187a1. 00000800:00000200:0.0:1784948851.316719:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948851.316725:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948851.316730:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b84187a1 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784948851.316851:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948851.316852:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948851.316863:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b84187a1. 00000800:00000001:3.0:1784948851.316869:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948851.316871:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948851.316882:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948851.316888:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948851.316900:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948851.316961:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948851.316972:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948851.316989:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948851.317010:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8541 00000800:00000001:3.0:1784948851.317021:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948851.317036:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948851.317041:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948851.317047:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948851.317056:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:3.0:1784948851.317060:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000400:00000200:3.0:1784948851.317066:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948851.317072:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000090a03b90. 00000400:00000010:3.0:1784948851.317088:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000378c7327. 00000800:00000001:3.0:1784948851.317100:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948851.317105:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948851.956033:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948851.956043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948851.956050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948851.956052:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948851.956144:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948851.956152:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948851.956157:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948852.148034:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948852.148042:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948852.148052:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948852.148058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948852.148063:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784948852.148066:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000069457997. 00000100:00000001:0.0:1784948852.148070:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948852.148074:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948852.148081:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948852.148083:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948852.148085:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948852.148098:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948852.148104:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948852.148114:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948852.148130:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000004c284093. 00000100:00000001:1.0:1784948852.148144:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948852.148161:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948852.148209:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948852.148214:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948852.148225:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069457997] to pc [ptlrpcd_00_03+3] req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948852.148277:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948852.148283:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948852.148290:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948852.148295:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948852.148302:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948852.148309:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005f948c2e. 00000100:00000001:0.0:1784948852.148313:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948852.148316:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948852.148319:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948852.148328:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948852.148328:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948852.148337:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1784948852.148339:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948852.148344:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.148362:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:1.0:1784948852.148362:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:0.0:1784948852.148367:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948852.148374:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948852.148375:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948852.148382:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948852.148388:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948852.148398:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289657216:0@lo:400: 00000100:00000040:1.0:1784948852.148398:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948852.148411:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948852.148421:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948852.148426:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.148426:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948852.148430:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1784948852.148436:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948852.148437:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948852.148439:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948852.148448:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948852.148454:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948852.148460:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005f948c2e] to pc [ptlrpcd_00_00+0] req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948852.148466:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bb684072. 00000400:00000010:0.0:1784948852.148481:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00000100:00000200:0.0:1784948852.148493:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657216, portal 25 00000100:00000001:0.0:1784948852.148498:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948852.148500:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549155328 : -115544160396288 : ffff96e9c75d1c00) 00000100:00000040:0.0:1784948852.148509:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948852.148520:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948852.148529:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948852.148532:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000089c8262f. 00000100:00000001:1.0:1784948852.148535:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948852.148538:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289657216, offset 0 00000100:00000001:3.0:1784948852.148545:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948852.148553:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948852.148557:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11129) 00000400:00000200:0.0:1784948852.148559:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948852.148560:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948852.148576:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948852.148590:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784948852.148609:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948852.148612:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948852.148617:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784948852.148618:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6436f80 02000000:00000001:3.0:1784948852.148624:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948852.148633:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7ba5 [2] + 0 02000000:00000001:3.0:1784948852.148634:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948852.148646:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bcdfbf21. 00000100:00100000:3.0:1784948852.148648:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005f948c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289657344:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948852.148657:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:0.0:1784948852.148658:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006ce54ab7 02000000:00000001:3.0:1784948852.148663:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784948852.148668:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948852.148668:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948852.148671:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784948852.148677:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948852.148678:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948852.148687:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000010:3.0:1784948852.148689:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948852.148696:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948852.148697:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006ce54ab7 00000400:00000010:0.0:1784948852.148702:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000006ce54ab7. 00000400:00000010:3.0:1784948852.148703:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f13fb872. 00000100:00000001:0.0:1784948852.148710:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000400:00000010:3.0:1784948852.148715:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000200:0.0:1784948852.148716:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948852.148727:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000000086d093 x1871651289657216 msgsize 224 00000100:00000200:3.0:1784948852.148729:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657344, portal 10 00000100:00000001:3.0:1784948852.148737:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1784948852.148744:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784948852.148744:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549150848 : -115544160400768 : ffff96e9c75d0a80) 00000100:00000040:0.0:1784948852.148751:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:3.0:1784948852.148762:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948852.148773:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948852.148775:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948852.148782:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948852.148786:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000400:00000200:0.0:1784948852.148788:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000400:00000010:0.0:1784948852.148793:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000100:00000001:0.0:1784948852.148799:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784948852.148802:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289657344, offset 0 00000100:00000200:0.0:1784948852.148807:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948852.148819:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:3.0:1784948852.148824:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:2.0:1784948852.148827:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657216 02000000:00000001:2.0:1784948852.148837:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948852.148840:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948852.148842:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948852.148849:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948852.148850:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:0.0:1784948852.148851:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948852.148859:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948852.148868:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:2.0:1784948852.148872:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657216 00000020:00000001:2.0:1784948852.148881:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:3.0:1784948852.148883:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000040:2.0:1784948852.148887:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784948852.148896:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784948852.148900:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948852.148947:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948852.148952:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784948852.148956:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:2.0:1784948852.148961:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:3.0:1784948852.148965:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3301:3301:248:4294967295] 192.168.202.137@tcp LPNI seq info [2144:2144:8:4294967295] 00000020:00000001:2.0:1784948852.148971:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784948852.148978:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948852.148985:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948852.148994:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000ed785a7. 00000400:00000200:3.0:1784948852.148995:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784948852.149003:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1784948852.149014:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000010:2.0:1784948852.149015:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000ed785a7. 00000100:00000001:0.0:1784948852.149017:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948852.149030:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000100:00000001:0.0:1784948852.149031:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305791531 00000020:00000010:2.0:1784948852.149032:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf1bf81. 00000800:00000200:3.0:1784948852.149049:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000010:0.0:1784948852.149050:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000020:00000010:2.0:1784948852.149050:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000bec1fbd. 00000020:00000010:2.0:1784948852.149065:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000001:0.0:1784948852.149067:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000800:00000200:3.0:1784948852.149077:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:2.0:1784948852.149084:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000200:3.0:1784948852.149085:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948852.149086:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305791531 00000100:00000001:2.0:1784948852.149095:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:0.0:1784948852.149101:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:2.0:1784948852.149102:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948852.149108:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.149112:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948852.149114:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.149121:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305791531 00000100:00000001:2.0:1784948852.149122:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948852.149124:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2063 usecs (3 jiffies) 00000100:00000010:0.0:1784948852.149133:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:3.0:1784948852.149136:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305791492 00000100:00000001:0.0:1784948852.149141:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948852.149143:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305791531 00000100:00000001:2.0:1784948852.149148:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.149151:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305791492 00000100:00000001:0.0:1784948852.149153:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305791531 00000100:00000001:2.0:1784948852.149158:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948852.149161:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305791531 00000100:00000010:0.0:1784948852.149163:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:2.0:1784948852.149164:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948852.149176:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305791492 00000100:00000001:0.0:1784948852.149177:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00100000:2.0:1784948852.149204:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 719 00000100:00000001:0.0:1784948852.149210:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948852.149215:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305791531 00000100:00000040:2.0:1784948852.149217:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948852.149218:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948852.149229:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782152784 : -115538927398832 : ffff96eaff465050) 00000100:00000001:3.0:1784948852.149231:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305791492 00000100:00000001:0.0:1784948852.149232:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.149240:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.149243:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305791531 00000100:00000040:2.0:1784948852.149245:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000086d093 x1871651289657216/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:513/0 lens 224/0 e 0 to 0 dl 1784948863 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948852.149252:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948852.149258:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948852.149260:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948852.149273:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948852.149275:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.149280:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948852.149287:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.149290:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.149296:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.149305:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948852.149319:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948852.149322:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784948852.149325:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000ed785a7. 00000100:00000001:3.0:1784948852.149328:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948852.149334:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948852.149336:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948852.149343:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:2.0:1784948852.149344:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000ed785a7. 00000100:00000001:3.0:1784948852.149349:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.149357:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948852.149359:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000086d093 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289657216:12345-0@lo:400: 00000100:00000200:2.0:1784948852.149366:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657216 00000020:00000001:2.0:1784948852.149370:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948852.149376:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948852.149380:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948852.149383:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948852.149386:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948852.149391:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948852.149394:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948852.149397:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948852.149401:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948852.149404:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948852.149407:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948852.149410:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948852.149414:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948852.149419:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000004d27d87b. 02000000:00000001:2.0:1784948852.149423:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948852.149425:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948852.149428:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948852.149431:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657216 00010000:00000001:2.0:1784948852.149435:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948852.149442:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000086d093 x1871651289657216/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:513/0 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948852.149448:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948852.149458:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00010000:00000001:2.0:1784948852.149459:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948852.149466:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948852.149468:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00001000:2.0:1784948852.149471:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=24 v=5 (1 1 1 1) 00000400:00000200:0.0:1784948852.149483:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948852.149484:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:1.0:1784948852.149487:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948852.149492:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000100:00000040:2.0:1784948852.149495:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000010:0.0:1784948852.149498:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:2.0:1784948852.149504:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:0.0:1784948852.149508:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:1.0:1784948852.149509:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948852.149511:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948852.149516:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948852.149519:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948852.149523:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948852.149524:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:1.0:1784948852.149537:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948852.149538:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948852.149546:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000100:00000001:0.0:1784948852.149553:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:2.0:1784948852.149556:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289657216, offset 224 00000100:00000040:0.0:1784948852.149566:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948852.149571:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948852.149574:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948852.149595:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948852.149607:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948852.149612:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437000 00000100:00000001:0.0:1784948852.149614:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784948852.149621:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6436f80 00000400:00000200:1.0:1784948852.149639:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 156720 00000800:00000001:1.0:1784948852.149653:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948852.149654:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8545 [1] + 224 00000400:00000200:2.0:1784948852.149667:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948852.149670:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948852.149676:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948852.149683:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948852.149686:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948852.149697:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948852.149704:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948852.149724:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:2.0:1784948852.149725:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948852.149731:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948852.149749:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c73d371c. 00000100:00000040:1.0:1784948852.149757:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000c73d371c x1871651289657344 msgsize 224 00000100:00000001:2.0:1784948852.149762:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1784948852.149773:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784948852.149776:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000010:2.0:1784948852.149781:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000100:00000001:2.0:1784948852.149786:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948852.149788:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948852.149795:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948852.149800:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784948852.149803:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948852.149806:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948852.149808:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948852.149809:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948852.149810:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948852.149817:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784948852.149823:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000086d093 x1871651289657216/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:513/0 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784948852.149827:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:1.0:1784948852.149830:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:0.0:1784948852.149834:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948852.149841:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bb684072. 00000800:00000001:1.0:1784948852.149844:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948852.149851:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdaa2431 00000100:00100000:2.0:1784948852.149854:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000086d093 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289657216:12345-0@lo:400: Request processed in 529us (1120us total) trans 0 rc 0/0 00000400:00000010:0.0:1784948852.149859:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdaa2431. 00000100:00000001:0.0:1784948852.149871:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784948852.149872:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 719 00000100:00000040:2.0:1784948852.149881:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:0.0:1784948852.149886:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948852.149888:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948852.149892:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948852.149899:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000bec1fbd. 00000020:00000010:2.0:1784948852.149946:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948852.149956:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcf1bf81. 00000100:00000001:3.0:1784948852.149968:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000040:2.0:1784948852.149969:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:0.0:1784948852.149970:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948852.149976:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657344 02000000:00000001:2.0:1784948852.149980:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:3.0:1784948852.149985:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000010:2.0:1784948852.149987:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000004d27d87b. 00000100:00000001:3.0:1784948852.149992:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:2.0:1784948852.149999:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948852.150000:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948852.150007:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948852.150013:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.150014:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:2.0:1784948852.150015:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948852.150021:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948852.150025:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657344 00000400:00000010:2.0:1784948852.150028:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ccad1f73. 00000100:00000001:0.0:1784948852.150029:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:3.0:1784948852.150030:0:342762:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1784948852.150036:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:3.0:1784948852.150038:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784948852.150042:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:2.0:1784948852.150043:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000085f9af2c. 00000020:00000001:3.0:1784948852.150044:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948852.150050:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948852.150054:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:0.0:1784948852.150056:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948852.150065:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00001000:0.0:1784948852.150068:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=24 v=5 (5 5 5 5) 00000020:00000001:3.0:1784948852.150071:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00001000:0.0:1784948852.150078:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=24 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948852.150080:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948852.150089:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948852.150092:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948852.150092:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d30e1e5e. 00000100:00000001:3.0:1784948852.150104:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1784948852.150105:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948852.150111:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948852.150113:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d30e1e5e. 00000100:00000001:0.0:1784948852.150116:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948852.150124:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948852.150125:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ba34905a. 00000020:00000010:3.0:1784948852.150137:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b08eb9bf. 00000100:00000001:0.0:1784948852.150146:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948852.150147:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:0.0:1784948852.150152:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948852.150156:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948852.150163:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948852.150166:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948852.150177:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00100000:0.0:1784948852.150209:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289657216:0@lo:400: 00000100:00000001:3.0:1784948852.150211:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948852.150220:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948852.150223:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948852.150231:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948852.150235:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000069457997 x1871651289657216/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948852.150237:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948852.150246:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948852.150251:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784948852.150253:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948852.150258:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948852.150263:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000069d2b91d. 02000000:00000001:0.0:1784948852.150270:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948852.150273:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948852.150275:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948852.150277:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948852.150280:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000004c284093. 00000100:00000001:3.0:1784948852.150282:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948852.150286:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948852.150291:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948852.150293:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000069457997. 00000100:00000001:0.0:1784948852.150298:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948852.150301:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948852.150304:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.150306:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948852.150310:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948852.150313:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948852.150315:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948852.150323:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948852.150326:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948852.150326:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784948852.150331:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 695 00000100:00000001:2.0:1784948852.150333:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:3.0:1784948852.150343:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948852.150347:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549157120 : -115544160394496 : ffff96e9c75d2300) 00000100:00000001:2.0:1784948852.150349:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948852.150355:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948852.150360:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000c73d371c x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:513/0 lens 224/0 e 0 to 0 dl 1784948863 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948852.150384:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948852.150389:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d30e1e5e. 00000100:00000001:3.0:1784948852.150392:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948852.150394:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d30e1e5e. 00000100:00100000:3.0:1784948852.150401:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000c73d371c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289657344:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784948852.150405:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657344 00000020:00000001:3.0:1784948852.150408:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948852.150412:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948852.150414:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948852.150417:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948852.150419:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784948852.150423:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948852.150426:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948852.150428:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948852.150430:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948852.150432:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948852.150435:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948852.150439:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948852.150441:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948852.150446:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000900bf136. 02000000:00000001:3.0:1784948852.150448:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.150451:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948852.150453:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948852.150456:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657344 00010000:00000001:3.0:1784948852.150459:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948852.150462:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000c73d371c x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:513/0 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948852.150472:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948852.150475:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948852.150477:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=19 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948852.150482:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948852.150485:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948852.150488:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784948852.150492:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948852.150494:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.150496:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948852.150499:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948852.150503:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00000200:3.0:1784948852.150507:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289657344, offset 224 00000400:00000200:3.0:1784948852.150513:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948852.150522:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948852.150529:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3302:3302:248:4294967295] 192.168.202.137@tcp LPNI seq info [2145:2145:8:4294967295] 00000400:00000200:3.0:1784948852.150540:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948852.150546:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948852.150550:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784948852.150556:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948852.150562:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948852.150566:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948852.150588:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948852.150592:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948852.150595:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948852.150596:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948852.150599:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948852.150605:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c73d371c x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:513/0 lens 224/224 e 0 to 0 dl 1784948863 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948852.150618:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000c73d371c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289657344:12345-192.168.202.137@tcp:400: Request processed in 232us (858us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948852.150628:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 695 00000100:00000040:3.0:1784948852.150632:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948852.150635:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948852.150638:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948852.150642:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b08eb9bf. 00000020:00000010:3.0:1784948852.150646:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000020:00000010:3.0:1784948852.150651:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000ba34905a. 00000020:00000040:3.0:1784948852.150657:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948852.150661:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948852.150720:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948852.150727:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:2.0:1784948852.150734:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948852.150744:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948852.150750:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:2.0:1784948852.150753:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:2.0:1784948852.150758:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948852.150761:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948852.150771:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948852.150782:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948852.150787:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948852.150793:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948852.150806:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948852.150822:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437000 00000400:00000200:2.0:1784948852.150832:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x854d [1] + 224 00000800:00000001:2.0:1784948852.150840:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948852.150853:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948852.150858:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948852.150864:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948852.150869:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948852.150874:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948852.150885:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948852.150937:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948852.150946:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948852.150952:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.150963:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948852.150968:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948852.150975:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948852.150977:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948852.150980:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f13fb872. 00000400:00000200:3.0:1784948852.150988:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:3.0:1784948852.150991:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00000001:3.0:1784948852.150994:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948852.151000:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948852.151012:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948852.151021:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948852.151023:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.151025:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948852.151027:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948852.151029:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948852.151031:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948852.151033:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948852.151039:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=19 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948852.151042:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=19 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948852.151045:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948852.151047:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948852.151049:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948852.151053:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948852.151061:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.151064:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948852.151066:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948852.151068:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948852.151070:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948852.151074:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948852.151082:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948852.151084:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948852.151086:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948852.151089:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948852.151097:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005f948c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289657344:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948852.151102:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.151104:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948852.151107:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005f948c2e x1871651289657344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948852.151114:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948852.151116:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948852.151118:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000eb3e5fa2. 02000000:00000001:3.0:1784948852.151122:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948852.151123:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948852.151125:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948852.151127:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948852.151129:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 02000000:00000001:3.0:1784948852.151132:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948852.151134:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948852.151135:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005f948c2e. 00000100:00000001:3.0:1784948852.151139:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948852.151140:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.151143:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948852.151147:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948852.151149:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948852.151151:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948852.151153:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948852.192001:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305791531, at 4305791532 00000100:00000010:1.0:1784948852.192017:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:0.0:1784948852.192055:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305791531, at 4305791532 00000100:00000040:3.0:1784948852.192064:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305791531, at 4305791532 00000100:00000010:0.0:1784948852.192070:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:2.0:1784948852.192073:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305791531, at 4305791532 00000100:00000010:3.0:1784948852.192080:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:2.0:1784948852.192088:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000800:00000100:0.0:1784948852.340024:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:0.0:1784948852.340032:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:0.0:1784948852.340043:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948852.340052:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948852.340062:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11125 health :900 00000400:00000200:0.0:1784948852.340071:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948852.340078:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:0.0:1784948852.340083:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000800:00000100:0.0:1784948852.340127:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.26@tcp 00000800:00000010:0.0:1784948852.340130:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:0.0:1784948852.340134:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948852.340138:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.26@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948852.340143:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948852.340147:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.26@tcp: 1 00000400:00000200:0.0:1784948852.340153:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.26@tcp: -114 00000400:00000200:2.0:1784948852.340174:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948852.340216:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:2.0:1784948852.340227:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.340231:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.340235:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b039801a 00000400:00000010:2.0:1784948852.340240:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b039801a. 00000400:00000200:2.0:1784948852.340248:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.340255:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:2.0:1784948852.340261:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fff5e130. 00000400:00000200:2.0:1784948852.340270:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:2.0:1784948852.340275:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:2.0:1784948852.340283:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000400:00000200:2.0:1784948852.340289:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.26@tcp(00000000a7af9ab9) state 0x840 00000400:00000001:2.0:1784948852.340294:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.340296:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.340299:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000400:00000010:2.0:1784948852.340302:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000400:00000200:2.0:1784948852.340307:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.340312:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.26@tcp: -114 00000400:00000010:2.0:1784948852.340316:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf393d23. 00000400:00000200:2.0:1784948852.340323:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.26@tcp:-110 00000400:00000200:2.0:1784948852.340329:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.26@tcp(00000000a7af9ab9) state 0x2040 rc -110 00000400:00000200:2.0:1784948852.340335:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.26@tcp 00000400:00000200:3.0:1784948852.340460:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.26@tcp NID 111.222.173.26@tcp: -110. discovery complete 00000400:00000200:3.0:1784948852.340469:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.173.26@tcp primary NID 111.222.173.26@tcp rc -110 00000100:00000010:3.0:1784948852.340490:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000d30e1e5e. 00000100:00000001:3.0:1784948852.340523:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784948852.340526:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000d30e1e5e refcount 1 to 111.222.173.26@tcp 00010000:00000010:3.0:1784948852.340532:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 000000008afa02c9. 00010000:00080000:3.0:1784948852.340584:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.173.26@tcp at tail 00010000:00000001:3.0:1784948852.340588:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948852.340592:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948852.340606:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 000000009af7b278. 00000100:00000400:3.0:1784948852.340612:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.142.26@tcp: rc = -75 00000100:00000040:3.0:1784948852.340615:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.142.26@tcp cnt=32 00000400:00000200:3.0:1784948852.340619:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784948852.340622:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784948852.340627:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784948852.340630:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784948852.340633:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784948852.340644:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784948852.340652:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.340657:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340662:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784948852.340665:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784948852.340671:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.340681:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:3.0:1784948852.340684:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.340687:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.340692:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340695:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784948852.340699:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784948852.340704:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.340729:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784948852.340733:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.340736:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.340740:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340746:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784948852.340748:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784948852.340752:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.340757:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784948852.340759:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.340761:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.340764:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340767:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784948852.340769:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:3.0:1784948852.340773:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.340781:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:3.0:1784948852.340783:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.340784:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.340788:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340790:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784948852.340791:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784948852.340797:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.340804:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948852.340808:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784948852.340815:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.340820:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948852.340829:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784948852.340829:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340837:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:3.0:1784948852.340843:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000200:3.0:1784948852.340847:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948852.340850:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f1904ca. 00000400:00000200:3.0:1784948852.340854:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000400:00000200:3.0:1784948852.340861:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.340863:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784948852.340866:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000400:00000200:3.0:1784948852.340866:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340872:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:3.0:1784948852.340877:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:3.0:1784948852.340882:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948852.340885:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000056670828. 00000400:00000200:3.0:1784948852.340889:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784948852.340895:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.340898:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948852.340941:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784948852.340955:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.340961:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784948852.340963:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:3.0:1784948852.340968:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.340981:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:2.0:1784948852.340991:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.340999:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.341006:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.341012:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:2.0:1784948852.341022:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3303:3303:250:4294967295] 111.222.142.1@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:2.0:1784948852.341035:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:2.0:1784948852.341050:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.341063:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784948852.341066:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:3.0:1784948852.341073:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.341076:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000010:2.0:1784948852.341077:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009bb74647. 00000400:00000200:3.0:1784948852.341080:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341089:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784948852.341091:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000200:3.0:1784948852.341100:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000800:00000200:2.0:1784948852.341102:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784948852.341108:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:3.0:1784948852.341114:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.341117:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.1:1784948852.341125:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784948852.341126:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341138:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784948852.341140:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:3.0:1784948852.341145:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.341149:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000200:3.0:1784948852.341151:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.341152:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.341155:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341157:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000200:3.0:1784948852.341159:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000200:3.0:1784948852.341165:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.341170:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:3.0:1784948852.341209:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:3.0:1784948852.341216:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948852.341218:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:3.0:1784948852.341222:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.341233:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341237:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784948852.341239:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:3.0:1784948852.341246:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.341253:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784948852.341260:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:3.0:1784948852.341266:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.341271:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.341274:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000010:2.0:1784948852.341275:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dcf1bf81. 00000400:00000200:3.0:1784948852.341279:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:3.0:1784948852.341285:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000010:2.0:1784948852.341287:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000200:3.0:1784948852.341291:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.341303:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:3.0:1784948852.341304:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.341309:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784948852.341310:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000a5d96d2. 00000400:00000200:3.0:1784948852.341320:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341325:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:2.0:1784948852.341332:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784948852.341334:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:3.0:1784948852.341346:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.341355:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:2.0:1784948852.341363:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.341369:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.341373:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.341379:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:2.0:1784948852.341387:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3304:3304:249:4294967295] 111.222.173.2@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:2.0:1784948852.341399:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:2.0:1784948852.341413:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.341426:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784948852.341432:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784948852.341436:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948852.341438:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007b2ad4bf. 00000400:00000200:3.0:1784948852.341440:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.341450:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341453:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000800:00000200:2.0:1784948852.341461:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784948852.341462:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000800:00000200:2.0:1784948852.341474:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948852.341475:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.341490:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784948852.341498:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.1:1784948852.341499:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784948852.341505:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.1:1784948852.341511:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948852.341518:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341528:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:3.0:1784948852.341536:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000800:00000100:2.0:1784948852.341537:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.2@tcp 00000800:00000010:2.0:1784948852.341545:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007b2ad4bf. 00000400:00000200:3.0:1784948852.341546:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.341561:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948852.341565:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000400:00000200:3.0:1784948852.341571:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.341577:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948852.341582:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948852.341590:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341596:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:3.0:1784948852.341598:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:2.0:1784948852.341599:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948852.341607:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.341614:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:3.0:1784948852.341622:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784948852.341629:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948852.341648:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:3.0:1784948852.341658:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.341669:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341676:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784948852.341680:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:2.0:1784948852.341690:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:3.0:1784948852.341692:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.341704:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 rc 0 00000400:00000200:3.0:1784948852.341711:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:3.0:1784948852.341718:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.341724:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948852.341729:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784948852.341739:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.341747:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:3.0:1784948852.341751:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000010:2.0:1784948852.341754:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009802f36f. 00000400:00000200:3.0:1784948852.341763:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948852.341766:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:3.0:1784948852.341777:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:3.0:1784948852.341783:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948852.341790:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000063d06960. 00000400:00000200:3.0:1784948852.341791:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.341808:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:2.0:1784948852.341809:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784948852.341820:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784948852.341829:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:2.0:1784948852.341832:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:3.0:1784948852.341842:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.341845:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.341856:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.341861:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.341866:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:2.0:1784948852.341872:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3305:3305:249:4294967295] 111.222.173.3@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1784948852.341880:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:2.0:1784948852.341890:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.341900:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:2.0:1784948852.341943:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007b2ad4bf. 00000400:00000200:3.0:1784948852.341953:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:3.0:1784948852.341960:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784948852.341961:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784948852.341968:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.1:1784948852.341976:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784948852.341983:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:2.0:1784948852.341993:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784948852.341995:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:2.0:1784948852.342004:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:3.0:1784948852.342009:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:2.0:1784948852.342018:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:3.0:1784948852.342024:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948852.342034:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000043c3e1d8. 00000400:00000200:3.0:1784948852.342042:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000010:2.0:1784948852.342045:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000400:00000200:3.0:1784948852.342050:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948852.342060:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006b55cc75. 00000400:00000200:3.0:1784948852.342060:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.342071:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:2.0:1784948852.342080:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784948852.342084:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:3.0:1784948852.342093:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:2.0:1784948852.342099:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:3.0:1784948852.342108:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.342111:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.342121:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.342126:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.342130:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:2.0:1784948852.342136:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3306:3306:248:4294967295] 111.222.142.3@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1784948852.342145:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:2.0:1784948852.342156:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.342167:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:2.0:1784948852.342173:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000925a7f5a. 00000400:00000200:3.0:1784948852.342175:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784948852.342210:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342214:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948852.342219:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784948852.342237:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342245:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000800:00000200:2.1:1784948852.342247:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784948852.342252:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:3.0:1784948852.342262:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948852.342268:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784948852.342270:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342271:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.342275:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342279:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:2.0:1784948852.342285:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:3.0:1784948852.342287:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:3.0:1784948852.342298:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.342304:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x261 rc 0 00000400:00000200:3.0:1784948852.342309:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:3.0:1784948852.342316:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342324:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948852.342330:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784948852.342332:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342343:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:3.0:1784948852.342345:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000010:2.0:1784948852.342348:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fff5e130. 00000400:00000200:3.0:1784948852.342353:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948852.342362:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000200:3.0:1784948852.342367:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:3.0:1784948852.342373:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342377:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.342381:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342384:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000010:2.0:1784948852.342386:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c3236cd0. 00000400:00000200:3.0:1784948852.342388:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000200:3.0:1784948852.342400:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.342409:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784948852.342416:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784948852.342419:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342424:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.342427:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342429:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:3.0:1784948852.342431:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:2.0:1784948852.342439:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:3.0:1784948852.342440:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.342453:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.342460:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.342464:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.342469:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:2.0:1784948852.342472:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3307:3307:247:4294967295] 111.222.173.4@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1784948852.342480:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:2.0:1784948852.342488:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.342496:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000800:00000010:2.0:1784948852.342503:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009839174d. 00000400:00000200:3.0:1784948852.342504:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:3.0:1784948852.342513:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342521:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948852.342526:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:2.0:1784948852.342537:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948852.342537:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342550:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000200:3.0:1784948852.342556:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000800:00000200:2.1:1784948852.342561:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784948852.342570:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784948852.342571:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948852.342581:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:3.0:1784948852.342590:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000100:2.0:1784948852.342591:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.4@tcp 00000800:00000010:2.0:1784948852.342599:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009839174d. 00000400:00000200:3.0:1784948852.342602:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948852.342607:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948852.342616:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:2.0:1784948852.342625:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948852.342627:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:2.0:1784948852.342634:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948852.342636:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:2.0:1784948852.342644:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:3.0:1784948852.342652:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948852.342657:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:2.0:1784948852.342672:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:2.0:1784948852.342676:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 rc 0 00000400:00000200:3.0:1784948852.342684:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784948852.342690:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342697:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948852.342701:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000200:3.0:1784948852.342711:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342718:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784948852.342722:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000010:2.0:1784948852.342723:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000053ab16eb. 00000400:00000010:2.0:1784948852.342733:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000400:00000200:3.0:1784948852.342737:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948852.342741:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000735cb99e. 00000400:00000200:2.0:1784948852.342752:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:3.0:1784948852.342758:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000200:3.0:1784948852.342764:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948852.342770:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948852.342775:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948852.342779:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000400:00000200:2.0:1784948852.342786:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000100:00000010:3.0:1784948852.342786:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 000000009af7b278. 00000400:00000200:2.0:1784948852.342797:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.342808:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:3.0:1784948852.342808:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00010000:00000001:3.0:1784948852.342818:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000400:00000200:2.0:1784948852.342819:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948852.342828:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:2.0:1784948852.342830:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:2.0:1784948852.342840:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3308:3308:247:4294967295] 111.222.142.4@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784948852.342848:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:2.0:1784948852.342859:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.342869:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000400:00000010:3.0:1784948852.342875:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000ba34905a. 00000800:00000010:2.0:1784948852.342877:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009839174d. 00000400:00000200:3.0:1784948852.342890:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000ba34905a nid 111.222.142.26@tcp 00000800:00000200:2.0:1784948852.342898:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784948852.342943:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000010:3.0:1784948852.342947:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000d2d3ade0. 00000400:00000200:2.0:1784948852.342948:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:2.0:1784948852.342956:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x261 rc 0 00000400:00000200:3.0:1784948852.342962:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 00000000d2d3ade0 net tcp 00000400:00000200:2.0:1784948852.342969:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:3.0:1784948852.342976:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000eb3e5fa2. 00000400:00000010:2.0:1784948852.342979:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c49404b8. 00000400:00000010:2.0:1784948852.342987:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000400:00000200:3.0:1784948852.342988:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000eb3e5fa2 nid 111.222.142.26@tcp 00000400:00000010:2.0:1784948852.342997:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005547c433. 00000400:00000200:2.0:1784948852.343010:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:3.0:1784948852.343022:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.142.26@tcp NID 111.222.142.26@tcp flags 0x0 00000400:00000200:3.0:1784948852.343029:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.142.26@tcp: 0 00000400:00000200:2.0:1784948852.343043:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:2.0:1784948852.343048:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343051:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343054:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343057:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:2.0:1784948852.343061:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3309:3309:246:4294967295] 111.222.142.5@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784948852.343066:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:2.0:1784948852.343073:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343079:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:2.0:1784948852.343084:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e6a3ade8. 00000400:00000200:3.0:1784948852.343086:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.26@tcp: 0 00000400:00000200:3.0:1784948852.343093:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784948852.343095:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784948852.343103:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:2.0:1784948852.343123:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:2.0:1784948852.343128:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:2.0:1784948852.343138:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:2.0:1784948852.343149:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000d531934e. 00000400:00000010:2.0:1784948852.343154:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000010:2.0:1784948852.343160:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009e875900. 00000400:00000200:2.0:1784948852.343170:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:2.0:1784948852.343210:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:2.0:1784948852.343216:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343221:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343226:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343231:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:2.0:1784948852.343237:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3310:3310:245:4294967295] 111.222.173.6@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:2.0:1784948852.343245:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:2.0:1784948852.343254:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343260:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:2.0:1784948852.343265:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000263f8d59. 00000800:00000200:2.0:1784948852.343273:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.0:1784948852.343277:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948852.343283:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784948852.343286:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948852.343291:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.6@tcp 00000800:00000010:2.0:1784948852.343294:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000263f8d59. 00000400:00000200:2.0:1784948852.343298:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948852.343302:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948852.343306:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948852.343309:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:2.0:1784948852.343312:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:2.0:1784948852.343319:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:2.0:1784948852.343321:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 rc 0 00000400:00000200:2.0:1784948852.343326:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:2.0:1784948852.343333:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006200396a. 00000400:00000010:2.0:1784948852.343338:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000400:00000010:2.0:1784948852.343345:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000038aa1d2e. 00000400:00000200:2.0:1784948852.343352:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:2.0:1784948852.343358:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:2.0:1784948852.343362:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343367:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343371:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343374:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:2.0:1784948852.343378:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3311:3311:245:4294967295] 111.222.173.7@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:2.0:1784948852.343383:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:2.0:1784948852.343389:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343394:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:2.0:1784948852.343398:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000263f8d59. 00000800:00000200:2.0:1784948852.343406:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784948852.343412:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:2.0:1784948852.343417:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:2.0:1784948852.343421:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000400:00000200:2.0:1784948852.343424:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:2.0:1784948852.343430:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000061851253. 00000400:00000010:2.0:1784948852.343435:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000400:00000010:2.0:1784948852.343439:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000147b89d3. 00000400:00000200:2.0:1784948852.343443:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:2.0:1784948852.343449:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:2.0:1784948852.343453:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343458:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343461:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343465:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:2.0:1784948852.343468:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3312:3312:244:4294967295] 111.222.142.7@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784948852.343472:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:2.0:1784948852.343481:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343486:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:2.0:1784948852.343490:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000230b350e. 00000800:00000200:2.0:1784948852.343495:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784948852.343500:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:2.0:1784948852.343504:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:2.0:1784948852.343508:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:2.0:1784948852.343514:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:2.0:1784948852.343518:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000307ddfde. 00000400:00000010:2.0:1784948852.343521:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000400:00000010:2.0:1784948852.343524:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000067f2d25d. 00000400:00000200:2.0:1784948852.343530:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:2.0:1784948852.343538:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:2.0:1784948852.343542:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343547:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343552:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343555:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:2.0:1784948852.343561:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3313:3313:243:4294967295] 111.222.173.8@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784948852.343566:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:2.0:1784948852.343574:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343581:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784948852.343587:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000016632059. 00000800:00000200:2.0:1784948852.343592:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.0:1784948852.343595:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948852.343601:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784948852.343604:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948852.343609:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784948852.343611:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000016632059. 00000400:00000200:2.0:1784948852.343614:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948852.343618:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948852.343622:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948852.343625:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:2.0:1784948852.343629:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:2.0:1784948852.343634:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:2.0:1784948852.343637:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 rc 0 00000400:00000200:2.0:1784948852.343640:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:2.0:1784948852.343646:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000010:2.0:1784948852.343651:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000400:00000010:2.0:1784948852.343655:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000dceddabe. 00000400:00000200:2.0:1784948852.343659:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:2.0:1784948852.343667:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:2.0:1784948852.343671:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343675:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343677:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343679:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:2.0:1784948852.343683:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3314:3314:243:4294967295] 111.222.142.8@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:2.0:1784948852.343687:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:2.0:1784948852.343692:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343697:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784948852.343699:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000016632059. 00000800:00000200:2.0:1784948852.343703:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784948852.343708:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000400:00000200:2.0:1784948852.343731:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:2.0:1784948852.343734:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x261 rc 0 00000400:00000200:2.0:1784948852.343738:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:2.0:1784948852.343742:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000bbdbdf4. 00000400:00000010:2.0:1784948852.343745:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000010:2.0:1784948852.343751:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000756d087e. 00000400:00000200:2.0:1784948852.343754:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:2.0:1784948852.343759:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:2.0:1784948852.343763:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343765:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343768:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343771:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:2.0:1784948852.343774:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3315:3315:242:4294967295] 111.222.173.9@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:2.0:1784948852.343779:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:2.0:1784948852.343785:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343790:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:2.0:1784948852.343794:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006d8f9877. 00000800:00000200:2.0:1784948852.343801:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784948852.343809:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000400:00000200:2.0:1784948852.343816:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:2.0:1784948852.343820:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x261 rc 0 00000400:00000200:2.0:1784948852.343826:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:2.0:1784948852.343830:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.343832:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.343835:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:2.0:1784948852.343838:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:2.0:1784948852.343844:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.343849:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:2.0:1784948852.343855:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dcf1bf81. 00000400:00000200:2.0:1784948852.343861:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-5 00000400:00000200:2.0:1784948852.343866:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -5 00000400:00000200:2.0:1784948852.343872:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000400:00000200:2.0:1784948852.343877:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:2.0:1784948852.343885:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000467d4d37. 00000400:00000010:2.0:1784948852.343890:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000010:2.0:1784948852.343895:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000096383a5. 00000400:00000200:2.0:1784948852.343899:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:2.0:1784948852.343931:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:2.0:1784948852.343934:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.343938:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.343940:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.343945:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:2.0:1784948852.343951:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3316:3316:241:4294967295] 111.222.142.9@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:2.0:1784948852.343958:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:2.0:1784948852.343966:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.343972:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:2.0:1784948852.343976:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006aefc2f9. 00000800:00000200:2.0:1784948852.343982:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784948852.343987:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:2.0:1784948852.344004:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:2.0:1784948852.344008:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:2.0:1784948852.344012:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:2.0:1784948852.344016:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dcf1bf81. 00000400:00000010:2.0:1784948852.344019:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000010:2.0:1784948852.344023:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000004d10c17. 00000400:00000200:2.0:1784948852.344028:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:2.0:1784948852.344034:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:2.0:1784948852.344038:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344041:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344046:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344051:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:2.0:1784948852.344057:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3317:3317:240:4294967295] 111.222.173.10@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1784948852.344063:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:2.0:1784948852.344072:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344081:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784948852.344086:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d625f2e8. 00000800:00000200:2.0:1784948852.344092:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.0:1784948852.344095:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948852.344101:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784948852.344103:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948852.344110:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784948852.344113:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d625f2e8. 00000400:00000200:2.0:1784948852.344118:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948852.344123:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948852.344126:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948852.344131:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:2.0:1784948852.344136:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:2.0:1784948852.344148:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:2.0:1784948852.344152:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 rc 0 00000400:00000200:2.0:1784948852.344157:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:2.0:1784948852.344166:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004b465f54. 00000400:00000010:2.0:1784948852.344172:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000400:00000010:2.0:1784948852.344199:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003c8b2437. 00000400:00000200:2.0:1784948852.344206:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:2.0:1784948852.344212:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:2.0:1784948852.344215:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344218:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344222:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344225:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:2.0:1784948852.344229:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3318:3318:240:4294967295] 111.222.173.11@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1784948852.344234:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:2.0:1784948852.344241:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344246:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784948852.344251:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d625f2e8. 00000800:00000200:2.0:1784948852.344255:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784948852.344260:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:2.0:1784948852.344265:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:2.0:1784948852.344267:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:2.0:1784948852.344272:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:2.0:1784948852.344279:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000208850ab. 00000400:00000010:2.0:1784948852.344286:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000400:00000010:2.0:1784948852.344292:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a7ce911a. 00000400:00000200:2.0:1784948852.344296:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:2.0:1784948852.344301:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:2.0:1784948852.344306:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344311:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344314:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344317:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:2.0:1784948852.344320:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3319:3319:239:4294967295] 111.222.142.11@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:2.0:1784948852.344325:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:2.0:1784948852.344334:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344340:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:2.0:1784948852.344346:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c03b4f3a. 00000800:00000200:2.0:1784948852.344353:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784948852.344358:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:2.0:1784948852.344363:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:2.0:1784948852.344365:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:2.0:1784948852.344369:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:2.0:1784948852.344375:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000072ce2c8. 00000400:00000010:2.0:1784948852.344379:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000400:00000010:2.0:1784948852.344383:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003d75c981. 00000400:00000200:2.0:1784948852.344388:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:2.0:1784948852.344395:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:2.0:1784948852.344400:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344404:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344407:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344411:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:2.0:1784948852.344414:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3320:3320:238:4294967295] 111.222.173.12@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:2.0:1784948852.344423:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:2.0:1784948852.344429:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344437:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784948852.344441:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000762a0fe3. 00000800:00000200:2.0:1784948852.344446:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.0:1784948852.344449:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948852.344455:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784948852.344458:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948852.344464:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784948852.344466:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000762a0fe3. 00000400:00000200:2.0:1784948852.344471:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948852.344476:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948852.344479:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948852.344482:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:2.0:1784948852.344486:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:2.0:1784948852.344495:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:2.0:1784948852.344499:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 rc 0 00000400:00000200:2.0:1784948852.344504:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:2.0:1784948852.344509:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000063f28b6f. 00000400:00000010:2.0:1784948852.344513:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000400:00000010:2.0:1784948852.344517:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000383d6bd1. 00000400:00000200:2.0:1784948852.344521:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:2.0:1784948852.344525:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:2.0:1784948852.344528:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344531:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344533:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344537:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:2.0:1784948852.344541:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3321:3321:238:4294967295] 111.222.142.12@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:2.0:1784948852.344546:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:2.0:1784948852.344552:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344556:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784948852.344560:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000762a0fe3. 00000800:00000200:2.0:1784948852.344564:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784948852.344571:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:2.0:1784948852.344575:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:2.0:1784948852.344578:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:2.0:1784948852.344582:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:2.0:1784948852.344589:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001879354a. 00000400:00000010:2.0:1784948852.344592:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000010:2.0:1784948852.344596:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001f662712. 00000400:00000200:2.0:1784948852.344600:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:2.0:1784948852.344604:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:2.0:1784948852.344607:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344609:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344611:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344613:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:2.0:1784948852.344616:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3322:3322:237:4294967295] 111.222.173.13@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:2.0:1784948852.344620:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:2.0:1784948852.344626:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344630:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:2.0:1784948852.344633:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d5a127c9. 00000800:00000200:2.0:1784948852.344636:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784948852.344640:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:2.0:1784948852.344643:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:2.0:1784948852.344645:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:2.0:1784948852.344648:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:2.0:1784948852.344651:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dc7a25c6. 00000400:00000010:2.0:1784948852.344653:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000400:00000010:2.0:1784948852.344657:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bfef55e5. 00000400:00000200:2.0:1784948852.344660:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:2.0:1784948852.344665:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:2.0:1784948852.344667:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344670:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344672:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344674:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:2.0:1784948852.344677:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3323:3323:236:4294967295] 111.222.142.13@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:2.0:1784948852.344681:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:2.0:1784948852.344687:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344691:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:2.0:1784948852.344694:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000069a945c8. 00000800:00000200:2.0:1784948852.344698:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784948852.344702:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:2.0:1784948852.344718:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:2.0:1784948852.344721:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:2.0:1784948852.344727:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:2.0:1784948852.344732:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000059e5b358. 00000400:00000010:2.0:1784948852.344736:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000400:00000010:2.0:1784948852.344741:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000054872d18. 00000400:00000200:2.0:1784948852.344745:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:2.0:1784948852.344753:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:2.0:1784948852.344757:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344761:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344764:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344767:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:2.0:1784948852.344770:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3324:3324:235:4294967295] 111.222.173.14@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:2.0:1784948852.344775:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:2.0:1784948852.344787:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.344793:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784948852.344800:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000036e798e7. 00000800:00000200:2.0:1784948852.344809:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.0:1784948852.344813:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.14 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948852.344821:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784948852.344826:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.14 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948852.344833:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784948852.344837:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000036e798e7. 00000400:00000200:2.0:1784948852.344843:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948852.344849:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948852.344854:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948852.344858:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:2.0:1784948852.344863:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:2.0:1784948852.344873:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:2.0:1784948852.344879:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 rc 0 00000400:00000200:2.0:1784948852.344887:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:2.0:1784948852.344898:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000010:2.0:1784948852.344946:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000400:00000010:2.0:1784948852.344954:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b03d8085. 00000400:00000200:2.0:1784948852.344959:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:2.0:1784948852.344969:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:2.0:1784948852.344975:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.344980:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.344984:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.344988:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:2.0:1784948852.344994:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3325:3325:235:4294967295] 111.222.142.14@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:2.0:1784948852.345000:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:2.0:1784948852.345008:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.345014:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:2.0:1784948852.345020:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000036e798e7. 00000800:00000200:2.0:1784948852.345027:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.1:1784948852.345034:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:2.0:1784948852.345041:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:2.0:1784948852.345044:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:2.0:1784948852.345049:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:2.0:1784948852.345054:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.345056:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.345059:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:2.0:1784948852.345061:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:2.0:1784948852.345067:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.345075:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:2.0:1784948852.345079:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fff5e130. 00000400:00000200:2.0:1784948852.345086:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-5 00000400:00000200:2.0:1784948852.345089:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -5 00000400:00000200:2.0:1784948852.345092:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:2.0:1784948852.345097:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:2.0:1784948852.345102:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fff5e130. 00000400:00000010:2.0:1784948852.345106:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000010:2.0:1784948852.345112:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000934e2586. 00000400:00000200:2.0:1784948852.345117:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:2.0:1784948852.345123:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:2.0:1784948852.345126:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.345129:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.345131:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.345134:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:2.0:1784948852.345138:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3326:3326:234:4294967295] 111.222.173.15@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:2.0:1784948852.345144:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:2.0:1784948852.345151:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.345156:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784948852.345160:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003ed79484. 00000800:00000200:2.0:1784948852.345164:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784948852.345170:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:2.0:1784948852.345174:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:2.0:1784948852.345199:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:2.0:1784948852.345204:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:2.0:1784948852.345213:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007a6587e5. 00000400:00000010:2.0:1784948852.345217:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000010:2.0:1784948852.345221:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d4c102f3. 00000400:00000200:2.0:1784948852.345225:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:2.0:1784948852.345230:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:2.0:1784948852.345233:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948852.345236:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.345239:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948852.345242:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:2.0:1784948852.345246:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3327:3327:233:4294967295] 111.222.142.15@tcp LPNI seq info [21:21:8:4294967295] 00000400:00000200:2.0:1784948852.345250:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:2.0:1784948852.345256:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.345260:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:2.0:1784948852.345264:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000057c7ab4. 00000800:00000200:2.0:1784948852.345269:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784948852.345274:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:2.0:1784948852.345290:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:2.0:1784948852.345293:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:2.0:1784948852.345297:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.26@tcp(00000000ba34905a) state 0x40 00000400:00000010:2.0:1784948852.345301:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003f3c01c8. 00000400:00000010:2.0:1784948852.345303:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000010:2.0:1784948852.345307:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001fcbecf4. 00000400:00000200:2.0:1784948852.345312:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.26@tcp 00000400:00000200:2.0:1784948852.345319:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.142.26@tcp local destination 00000400:00000200:2.0:1784948852.345324:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948852.345330:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.142.26@tcp 00000400:00000200:2.0:1784948852.345339:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.142.26@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:2.0:1784948852.345346:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3328:3328:232:4294967295] 111.222.142.26@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1784948852.345351:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.26@tcp 00000400:00000200:2.0:1784948852.345361:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.26@tcp(111.222.142.26@tcp:111.222.142.26@tcp) : GET try# 0 00000800:00000200:2.0:1784948852.345370:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.26@tcp 00000800:00000010:2.0:1784948852.345375:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009ff40a64. 00000800:00000010:2.0:1784948852.345383:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000007484db5a. 00000800:00000010:2.0:1784948852.345388:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000b05cdef8. 00000800:00000200:2.0:1784948852.345393:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007484db5a] -> 12345-111.222.142.26@tcp (2) 00000800:00000200:2.1:1784948852.345397:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007484db5a] -> 12345-111.222.142.26@tcp (2) 00000400:00000200:2.0:1784948852.345401:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.26@tcp 00000400:00000200:2.0:1784948852.345403:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.26@tcp(00000000ba34905a) state 0x240 rc 0 00000400:00000200:2.0:1784948852.345407:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000400:00000001:2.0:1784948852.345410:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.345413:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.345414:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:2.0:1784948852.345416:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000400:00000200:2.0:1784948852.345419:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.345422:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:2.0:1784948852.345426:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000d531934e. 00000400:00000200:2.0:1784948852.345430:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-5 00000400:00000200:2.0:1784948852.345433:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -5 00000400:00000200:2.0:1784948852.345436:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:2.0:1784948852.345440:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000001:2.0:1784948852.345443:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.345445:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.345446:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000400:00000010:2.0:1784948852.345448:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000400:00000200:2.0:1784948852.345453:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.345457:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:2.0:1784948852.345461:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000307ddfde. 00000400:00000200:2.0:1784948852.345468:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-5 00000400:00000200:2.0:1784948852.345471:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -5 00000400:00000200:2.0:1784948852.345475:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000400:00000200:2.0:1784948852.345482:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000001:2.0:1784948852.345485:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.345486:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.345487:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:2.0:1784948852.345489:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:2.0:1784948852.345491:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.345494:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:2.0:1784948852.345496:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dcf1bf81. 00000400:00000200:2.0:1784948852.345499:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-5 00000400:00000200:2.0:1784948852.345503:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -5 00000400:00000200:2.0:1784948852.345507:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00000400:00000200:2.0:1784948852.345514:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:2.0:1784948852.345519:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.345521:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.345524:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000010:2.0:1784948852.345526:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000400:00000200:2.0:1784948852.345529:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.345532:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:2.0:1784948852.345534:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000072ce2c8. 00000400:00000200:2.0:1784948852.345539:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-5 00000400:00000200:2.0:1784948852.345543:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -5 00000400:00000200:2.0:1784948852.345547:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000400:00000200:2.0:1784948852.345553:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000001:2.0:1784948852.345556:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948852.345558:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948852.345561:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:2.0:1784948852.345564:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000400:00000200:2.0:1784948852.345568:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948852.345571:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:2.0:1784948852.345575:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000059e5b358. 00000400:00000200:2.0:1784948852.345580:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-5 00000400:00000200:2.0:1784948852.345583:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -5 00000400:00000200:2.0:1784948852.345589:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00010000:00000001:2.0:1784948852.980014:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948852.980024:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948852.980032:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948852.980034:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948852.980168:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948852.980203:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948852.980208:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948853.172003:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948853.172020:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948853.172022:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948853.172028:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948853.172031:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948853.172035:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948853.172038:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948853.364119:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.2@tcp 00000800:00000010:0.0:1784948853.364127:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008f5595bf. 00000400:00000200:0.0:1784948853.364138:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948853.364146:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948853.364157:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948853.364162:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:0.0:1784948853.364204:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:1.0:1784948853.364247:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948853.364257:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000048fd39a3. 00000400:00000010:1.0:1784948853.364268:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000072ce5ef0. 00000400:00000200:2.0:1784948853.364283:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948853.364299:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2861 00000400:00000001:2.0:1784948853.364308:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948853.364310:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948853.364314:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:2.0:1784948853.364317:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000400:00000200:2.0:1784948853.364322:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948853.364327:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:2.0:1784948853.364332:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000011003cc7. 00000400:00000200:2.0:1784948853.364338:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-110 00000400:00000200:2.0:1784948853.364341:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -110 00000400:00000200:2.0:1784948853.364345:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000010:1.0:1784948853.364423:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000400:00000010:1.0:1784948853.364436:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008c245eeb. 00000400:00000200:1.0:1784948853.364444:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948853.364461:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948853.364469:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948853.364474:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3329:3329:232:4294967295] 192.168.202.137@tcp LPNI seq info [2146:2146:8:4294967295] 00000400:00000200:1.0:1784948853.364482:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948853.364491:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948853.364498:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948853.364503:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000800:00000200:1.0:1784948853.364511:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948853.364521:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948853.364526:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948853.364550:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11127 health :700 00000800:00000200:0.0:1784948853.364698:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948853.364705:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:0.0:1784948853.364713:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948853.364721:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784948853.364724:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948853.364727:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948853.364733:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784948853.364744:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948853.364751:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948853.364758:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948853.364769:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948853.364784:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948853.364795:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948853.364811:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948853.364820:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3330:3330:232:4294967295] 192.168.202.137@tcp LPNI seq info [2147:2147:8:4294967295] 00000400:00000200:2.0:1784948853.364836:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948853.364847:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948853.364853:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003a8a98bc. 00000800:00000200:2.0:1784948853.364864:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948853.364874:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948853.364879:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003a8a98bc type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784948853.364975:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948853.365047:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948853.365053:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000400:00000200:2.0:1784948853.365057:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948853.365064:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948853.365079:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948853.365087:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948853.365090:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948853.365096:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948853.365102:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000001:3.0:1784948853.365109:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948853.365121:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x85c9 00000800:00000001:2.0:1784948853.365128:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948853.365140:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948853.365143:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948853.365147:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948853.365152:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:2.0:1784948853.365155:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000400:00000200:2.0:1784948853.365158:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948853.365162:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000048fd39a3. 00000400:00000010:2.0:1784948853.365165:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000072ce5ef0. 00000800:00000001:2.0:1784948853.365196:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948853.365201:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948854.004023:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948854.004033:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948854.004042:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948854.004044:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948854.004136:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948854.004142:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948854.004146:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948854.196077:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948854.196095:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948854.196098:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948854.196105:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948854.196109:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948854.196114:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948854.196117:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948854.388055:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000800:00000010:0.0:1784948854.388063:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002a753d9d. 00000400:00000200:0.0:1784948854.388077:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948854.388085:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948854.388099:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11127 health :900 00000400:00000200:0.0:1784948854.388108:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948854.388115:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:0.0:1784948854.388121:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000800:00000100:0.0:1784948854.388234:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.6@tcp 00000800:00000010:0.0:1784948854.388238:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d104522b. 00000400:00000200:2.0:1784948854.388246:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948854.388248:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948854.388265:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2861 00000400:00000200:0.0:1784948854.388266:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000001:2.0:1784948854.388282:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948854.388287:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948854.388290:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:2.0:1784948854.388293:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000400:00000200:2.0:1784948854.388300:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948854.388302:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948854.388309:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:0.0:1784948854.388313:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000010:2.0:1784948854.388318:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000400:00000200:2.0:1784948854.388329:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:0.0:1784948854.388333:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:2.0:1784948854.388339:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:2.0:1784948854.388349:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000400:00000200:2.0:1784948854.388358:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2861 00000400:00000001:2.0:1784948854.388362:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948854.388364:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948854.388365:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000400:00000010:2.0:1784948854.388367:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000400:00000200:2.0:1784948854.388372:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948854.388378:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:2.0:1784948854.388382:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000684a4a3c. 00000400:00000200:2.0:1784948854.388387:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-110 00000400:00000200:2.0:1784948854.388389:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -110 00000400:00000200:2.0:1784948854.388392:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00010000:00000001:2.0:1784948855.027978:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948855.027985:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948855.027990:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948855.027991:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948855.028048:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948855.028052:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948855.028054:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948855.220009:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948855.220027:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948855.220030:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948855.220037:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948855.220041:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948855.220047:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948855.220050:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948855.412213:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948855.412223:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000eb33d71d. 00000400:00000010:1.0:1784948855.412238:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dd0652fb. 00000400:00000010:1.0:1784948855.412249:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000400:00000010:1.0:1784948855.412260:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000038468d93. 00000800:00000100:0.0:1784948855.412270:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.5@tcp 00000400:00000200:1.0:1784948855.412272:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948855.412288:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000062b4a223. 00000400:00000200:1.0:1784948855.412299:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948855.412302:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948855.412311:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948855.412319:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948855.412471:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3331:3331:234:4294967295] 192.168.202.137@tcp LPNI seq info [2148:2148:8:4294967295] 00000400:00000200:1.0:1784948855.412484:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948855.412492:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948855.412500:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948855.412509:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11128 health :700 00000800:00000010:1.0:1784948855.412509:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000400:00000200:0.0:1784948855.412525:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000200:1.0:1784948855.412526:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784948855.412542:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000800:00000200:1.0:1784948855.412549:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948855.412559:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1784948855.412566:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000800:00000100:0.0:1784948855.412630:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.10@tcp 00000800:00000010:0.0:1784948855.412637:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000034aacccc. 00000400:00000200:0.0:1784948855.412649:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948855.412662:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948855.412671:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948855.412677:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:0.0:1784948855.412684:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:3.0:1784948855.412726:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948855.412741:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2861 00000400:00000001:3.0:1784948855.412750:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948855.412752:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948855.412756:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000010:3.0:1784948855.412759:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000400:00000200:3.0:1784948855.412765:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948855.412770:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:3.0:1784948855.412775:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006b8bfa56. 00000400:00000200:3.0:1784948855.412786:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-110 00000400:00000200:3.0:1784948855.412790:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -110 00000400:00000200:3.0:1784948855.412794:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:3.0:1784948855.412799:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2861 00000400:00000001:3.0:1784948855.412804:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948855.412806:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948855.412807:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007d821c01 00000400:00000010:3.0:1784948855.412809:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007d821c01. 00000400:00000200:3.0:1784948855.412813:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948855.412816:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:3.0:1784948855.412819:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000898cc47a. 00000400:00000200:3.0:1784948855.412823:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-110 00000400:00000200:3.0:1784948855.412827:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -110 00000400:00000200:3.0:1784948855.412837:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000800:00000200:2.0:1784948855.412863:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948855.412877:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:2.0:1784948855.412891:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948855.412893:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948855.412951:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948855.412951:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:0.0:1784948855.412965:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948855.412974:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784948855.412976:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948855.412984:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:0.0:1784948855.412994:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948855.413015:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948855.413026:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948855.413038:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948855.413045:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3332:3332:236:4294967295] 192.168.202.137@tcp LPNI seq info [2149:2149:8:4294967295] 00000400:00000200:0.0:1784948855.413056:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948855.413062:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948855.413067:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b84187a1. 00000800:00000200:0.0:1784948855.413074:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948855.413081:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948855.413086:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b84187a1 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784948855.413143:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948855.413278:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948855.413286:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948855.413292:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948855.413293:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b84187a1. 00000800:00000001:2.0:1784948855.413304:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948855.413308:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948855.413311:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948855.413325:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948855.413335:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000001:3.0:1784948855.413353:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948855.413370:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x85cd 00000800:00000001:2.0:1784948855.413380:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948855.413396:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948855.413399:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948855.413404:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948855.413412:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000400:00000010:2.0:1784948855.413415:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000400:00000200:2.0:1784948855.413418:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948855.413421:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000eb33d71d. 00000400:00000010:2.0:1784948855.413425:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dd0652fb. 00000800:00000001:2.0:1784948855.413433:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948855.413435:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948856.052024:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948856.052034:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948856.052041:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948856.052042:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948856.052130:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948856.052136:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948856.052140:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948856.244005:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948856.244017:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948856.244019:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948856.244022:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948856.244025:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948856.244028:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948856.244029:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948857.076020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948857.076030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948857.076037:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948857.076038:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948857.076153:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948857.076158:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948857.076161:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948857.268075:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1784948857.268094:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948857.268100:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948857.268107:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948857.268109:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948857.268120:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784948857.268121:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000100:00000001:3.0:1784948857.268126:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1784948857.268133:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948857.268167:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948857.268175:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948857.268187:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948857.268194:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948857.268200:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948857.268213:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b84187a1. 00000100:00000001:0.0:1784948857.268222:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948857.268234:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948857.268258:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948857.268261:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948857.268267:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_00_01+1] req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948857.268304:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948857.268307:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948857.268312:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948857.268315:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948857.268319:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948857.268322:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000020:00000040:0.0:1784948857.268325:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948857.268327:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948857.268329:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948857.268332:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948857.268337:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000595fa8f8. 00000100:00000001:0.0:1784948857.268341:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948857.268348:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948857.268359:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948857.268362:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948857.268368:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c39699a8] to pc [ptlrpcd_00_02+2] req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948857.268372:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948857.268381:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948857.268389:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948857.268392:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.268398:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948857.268408:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948857.268418:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11134) 00000100:00000040:2.0:1784948857.268420:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948857.268445:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948857.268446:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948857.268449:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948857.268451:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948857.268455:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1784948857.268461:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948857.268463:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948857.268465:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948857.268469:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948857.268477:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948857.268481:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948857.268495:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1784948857.268497:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289657472:0@lo:400: 00000100:00000001:1.0:1784948857.268499:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948857.268506:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948857.268510:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948857.268510:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948857.268517:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948857.268521:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289657600:192.168.202.137@tcp:400: 02000000:00000001:2.0:1784948857.268525:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948857.268526:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948857.268533:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948857.268533:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948857.268541:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1784948857.268546:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948857.268550:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784948857.268554:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 02000000:00000010:1.0:1784948857.268556:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 02000000:00000001:1.0:1784948857.268562:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948857.268562:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948857.268577:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000502cbfd9. 00000400:00000010:2.0:1784948857.268578:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a0670fc. 00000400:00000010:1.0:1784948857.268589:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000400:00000010:2.0:1784948857.268591:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000100:00000200:1.0:1784948857.268602:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657600, portal 10 00000100:00000200:2.0:1784948857.268603:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657472, portal 25 00000100:00000001:1.0:1784948857.268609:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948857.268611:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948857.268616:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558302976 : -115544151248640 : ffff96e9c7e8b100) 00000100:00000001:2.0:1784948857.268617:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000040:2.0:1784948857.268631:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948857.268632:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948857.268677:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948857.268678:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948857.268683:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784948857.268688:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948857.268691:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00000200:1.0:1784948857.268696:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289657600, offset 0 00000400:00000010:2.0:1784948857.268696:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000100:00000200:2.0:1784948857.268706:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289657472, offset 0 00000400:00000200:1.0:1784948857.268712:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948857.268722:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948857.268737:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948857.268744:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948857.268750:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948857.268755:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948857.268758:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948857.268762:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3333:3333:236:4294967295] 192.168.202.137@tcp LPNI seq info [2150:2150:8:4294967295] 00000400:00000200:1.0:1784948857.268775:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948857.268782:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948857.268789:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 00000400:00000200:2.0:1784948857.268791:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:1.0:1784948857.268803:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784948857.268813:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437080 00000800:00000200:1.0:1784948857.268818:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948857.268826:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1784948857.268832:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7bc1 [2] + 0 00000400:00000010:2.0:1784948857.268840:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001a3c4d34. 00000100:00000001:1.0:1784948857.268847:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948857.268852:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000de41855a 00000100:00000040:1.0:1784948857.268855:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2065 usecs (3 jiffies) 00000100:00000001:1.0:1784948857.268861:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305796611 00000100:00000010:1.0:1784948857.268867:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000bbdc2de7. 00000400:00000200:2.0:1784948857.268869:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948857.268877:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000400:00000200:2.0:1784948857.268881:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948857.268891:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000de41855a 00000100:00000001:1.0:1784948857.268893:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305796611 00000400:00000010:2.0:1784948857.268896:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000de41855a. 00000100:00000010:1.0:1784948857.268898:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005975f076. 00000100:00000001:1.0:1784948857.268937:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948857.268942:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305796612 00000100:00000010:1.0:1784948857.268945:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000004f2eb99f. 00000100:00000001:1.0:1784948857.268951:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784948857.268959:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305796612 00000100:00000010:1.0:1784948857.268965:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:1.0:1784948857.268973:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948857.268973:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948857.268981:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784948857.268986:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948857.268989:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000ffe216dc x1871651289657472 msgsize 224 00000100:00000001:1.0:1784948857.268991:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948857.268999:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948857.269005:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948857.269006:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948857.269013:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948857.269016:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948857.269021:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948857.269027:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948857.269029:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948857.269036:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.269038:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948857.269039:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948857.269055:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000010:2.0:1784948857.269061:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000100:00000001:2.0:1784948857.269067:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948857.269074:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948857.269096:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948857.269099:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1784948857.269105:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657472 00000100:00000040:2.0:1784948857.269110:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948857.269114:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948857.269122:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:0.0:1784948857.269127:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948857.269130:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.269134:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.269162:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000010:0.0:1784948857.269166:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000800:00000001:1.0:1784948857.269172:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784948857.269174:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.269179:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948857.269182:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:3.0:1784948857.269186:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657472 00000800:00000001:1.0:1784948857.269192:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.269194:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305796651 00000020:00000001:3.0:1784948857.269197:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000001:1.0:1784948857.269202:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948857.269214:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000040:3.0:1784948857.269218:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000001:1.0:1784948857.269219:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.269225:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305796611 00000400:00000200:0.0:1784948857.269227:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000020:00000001:3.0:1784948857.269243:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784948857.269244:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000100:00000001:2.0:1784948857.269248:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305796651 00000400:00000200:1.0:1784948857.269260:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000040:3.0:1784948857.269261:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948857.269266:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948857.269266:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305796611 00000020:00000001:3.0:1784948857.269284:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948857.269288:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305796651 00000100:00000200:0.0:1784948857.269291:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948857.269302:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437100 00000020:00000001:3.0:1784948857.269311:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948857.269314:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305796612 00000100:00000001:2.0:1784948857.269335:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305796651 00000400:00000200:1.0:1784948857.269342:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 156944 00000100:00000001:3.0:1784948857.269344:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948857.269358:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305796612 00000020:00000010:3.0:1784948857.269364:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000594c0c3d. 00000800:00000001:1.0:1784948857.269380:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948857.269381:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948857.269384:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948857.269386:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784948857.269395:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.269395:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948857.269400:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000594c0c3d. 00000100:00000040:0.0:1784948857.269406:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948857.269409:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948857.269411:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948857.269429:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000898cc47a. 00000100:00000001:2.0:1784948857.269430:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948857.269445:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948857.269456:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:3.0:1784948857.269457:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b08eb9bf. 00000100:00000001:1.0:1784948857.269470:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784948857.269472:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000200:1.0:1784948857.269480:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000010:3.0:1784948857.269482:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:0.0:1784948857.269494:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.269496:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784948857.269497:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000db13d9bb. 00000100:00000001:0.0:1784948857.269509:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784948857.269511:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000db13d9bb x1871651289657600 msgsize 224 00000100:00000001:2.0:1784948857.269511:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948857.269513:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948857.269530:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948857.269537:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948857.269543:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948857.269546:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:1.0:1784948857.269551:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784948857.269554:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948857.269570:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948857.269576:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948857.269585:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948857.269596:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948857.269602:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948857.269642:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948857.269646:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:0.0:1784948857.269649:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657600 00000100:00000001:3.0:1784948857.269653:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:0.0:1784948857.269656:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948857.269661:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948857.269662:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948857.269670:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.269671:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948857.269674:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 720 02000000:00000001:0.0:1784948857.269682:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948857.269690:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657600 00000100:00000040:3.0:1784948857.269691:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948857.269692:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.269693:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948857.269701:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:2.0:1784948857.269709:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.269710:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948857.269710:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782151760 : -115538927399856 : ffff96eaff464c50) 00000020:00000040:0.0:1784948857.269719:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948857.269730:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948857.269735:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:3.0:1784948857.269738:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ffe216dc x1871651289657472/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:518/0 lens 224/0 e 0 to 0 dl 1784948868 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948857.269743:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948857.269751:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784948857.269762:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948857.269770:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784948857.269779:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948857.269781:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000020:00000010:3.0:1784948857.269792:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000594c0c3d. 00000020:00000010:0.0:1784948857.269793:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000003f616a. 00000020:00000010:0.0:1784948857.269804:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784948857.269806:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948857.269817:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000594c0c3d. 00000100:00000040:0.0:1784948857.269820:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948857.269826:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948857.269829:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:3.0:1784948857.269833:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ffe216dc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289657472:12345-0@lo:400: 00000100:00000001:0.0:1784948857.269835:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948857.269847:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:3.0:1784948857.269850:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657472 00000100:00000001:0.0:1784948857.269853:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948857.269859:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948857.269861:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948857.269870:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948857.269874:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948857.269881:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784948857.269882:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948857.269890:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784948857.269900:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000001:3.0:1784948857.269900:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948857.269905:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948857.269945:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948857.269956:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 696 00000100:00000001:2.0:1784948857.269959:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784948857.269963:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948857.269967:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948857.269970:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549149056 : -115544160402560 : ffff96e9c75d0380) 00000100:00000001:2.0:1784948857.269981:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948857.269982:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:0.0:1784948857.269987:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000db13d9bb x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:518/0 lens 224/0 e 0 to 0 dl 1784948868 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948857.269993:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:2.0:1784948857.269998:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948857.270006:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948857.270022:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948857.270027:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948857.270030:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000010:0.0:1784948857.270033:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 02000000:00000001:3.0:1784948857.270040:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784948857.270041:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948857.270050:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 02000000:00000010:3.0:1784948857.270054:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 00000100:00100000:0.0:1784948857.270056:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000db13d9bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289657600:12345-192.168.202.137@tcp:400: 02000000:00000001:3.0:1784948857.270064:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948857.270067:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657600 00000100:00000001:3.0:1784948857.270076:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.270077:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948857.270083:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948857.270086:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.270088:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.270096:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00010000:00000040:3.0:1784948857.270097:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657472 00000020:00000001:0.0:1784948857.270099:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948857.270108:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00010000:00000001:3.0:1784948857.270111:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:0.0:1784948857.270114:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948857.270121:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948857.270124:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00010000:00000200:3.0:1784948857.270130:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ffe216dc x1871651289657472/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:518/0 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948857.270131:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.270162:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948857.270169:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948857.270173:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948857.270181:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000faa299c5. 02000000:00000001:0.0:1784948857.270187:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948857.270189:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948857.270191:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.270199:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948857.270199:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948857.270212:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657600 00000100:00001000:3.0:1784948857.270213:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=29 v=5 (1 1 1 1) 00010000:00000001:0.0:1784948857.270216:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948857.270227:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000db13d9bb x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:518/0 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948857.270230:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948857.270242:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00010000:00000001:0.0:1784948857.270247:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948857.270251:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00010000:00000001:0.0:1784948857.270253:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948857.270263:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=24 v=5 (1 1 1 1) 02000000:00000001:3.0:1784948857.270266:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784948857.270270:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784948857.270274:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948857.270277:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948857.270285:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784948857.270287:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784948857.270290:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948857.270295:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948857.270304:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948857.270305:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948857.270314:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948857.270316:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000400:00000010:0.0:1784948857.270319:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00000100:00000200:3.0:1784948857.270329:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289657472, offset 224 00000100:00000200:0.0:1784948857.270331:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289657600, offset 224 00000400:00000200:3.0:1784948857.270357:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948857.270371:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948857.270383:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948857.270392:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948857.270402:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3334:3334:236:4294967295] 192.168.202.137@tcp LPNI seq info [2151:2151:8:4294967295] 00000400:00000200:0.0:1784948857.270416:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948857.270423:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948857.270430:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a3cf4688. 00000400:00000200:3.0:1784948857.270438:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437080 00000800:00000200:0.0:1784948857.270449:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948857.270465:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x85d5 [1] + 224 00000800:00000200:0.0:1784948857.270466:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948857.270477:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a3cf4688 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1784948857.270493:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948857.270499:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948857.270509:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:0.0:1784948857.270514:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948857.270521:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784948857.270524:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1784948857.270525:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948857.270534:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:3.0:1784948857.270543:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948857.270544:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db13d9bb x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:518/0 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948857.270615:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000db13d9bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289657600:12345-192.168.202.137@tcp:400: Request processed in 586us (1102us total) trans 0 rc 0/0 00000100:00000040:3.0:1784948857.270620:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948857.270637:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 696 00000100:00000040:0.0:1784948857.270647:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948857.270655:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948857.270660:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948857.270668:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000003f616a. 00000020:00000010:0.0:1784948857.270675:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948857.270679:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 00000020:00000040:0.0:1784948857.270684:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948857.270689:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948857.270690:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948857.270712:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000800:00000001:0.0:1784948857.270719:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784948857.270721:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00000001:3.0:1784948857.270731:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784948857.270735:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948857.270738:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948857.270742:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948857.270745:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948857.270755:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948857.270756:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784948857.270756:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a3cf4688. 00000100:00000040:3.0:1784948857.270775:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:2.0:1784948857.270778:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948857.270780:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948857.270793:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948857.270797:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:3.0:1784948857.270802:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:1.0:1784948857.270809:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948857.270815:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1784948857.270815:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948857.270819:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437100 00000400:00000200:2.0:1784948857.270826:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:1.0:1784948857.270827:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a0670fc. 00000020:00000001:3.0:1784948857.270831:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:1.0:1784948857.270843:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:1.0:1784948857.270848:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000100:00000040:3.0:1784948857.270853:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ffe216dc x1871651289657472/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:518/0 lens 224/224 e 0 to 0 dl 1784948868 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948857.270856:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1784948857.270866:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x85d1 [1] + 224 00000100:00000200:1.0:1784948857.270871:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948857.270900:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948857.270901:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdaa2431 00000100:00000200:1.0:1784948857.270941:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948857.270959:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdaa2431. 00000800:00000001:0.0:1784948857.270965:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.270977:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:0.0:1784948857.270985:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948857.270988:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948857.270989:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948857.270997:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948857.271009:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948857.271013:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00100000:3.0:1784948857.271013:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ffe216dc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289657472:12345-0@lo:400: Request processed in 1214us (2002us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948857.271020:0:337114:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1784948857.271021:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948857.271030:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:0.0:1784948857.271039:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948857.271041:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948857.271058:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948857.271066:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 720 00000100:00001000:1.0:1784948857.271077:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=29 v=5 (5 5 5 5) 00000100:00000040:3.0:1784948857.271083:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948857.271103:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00001000:1.0:1784948857.271109:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=29 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948857.271113:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:0.0:1784948857.271118:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948857.271128:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:3.0:1784948857.271131:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b08eb9bf. 00000100:00000001:1.0:1784948857.271134:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948857.271155:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784948857.271160:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948857.271176:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948857.271182:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000100:00000040:1.0:1784948857.271199:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948857.271200:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000898cc47a. 00000100:00000001:0.0:1784948857.271210:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948857.271226:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1784948857.271227:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000800:00000001:0.0:1784948857.271229:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948857.271243:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000001:0.0:1784948857.271245:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948857.271248:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948857.271249:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948857.271262:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 00000100:00000040:1.0:1784948857.271263:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948857.271280:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948857.271282:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00100000:1.0:1784948857.271284:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289657472:0@lo:400: 00000100:00000001:2.0:1784948857.271291:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948857.271294:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948857.271297:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1784948857.271301:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784948857.271310:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000001:2.0:1784948857.271311:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:3.0:1784948857.271316:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:1.0:1784948857.271318:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b3c18d2b x1871651289657472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948857.271318:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000502cbfd9. 00000100:00000001:1.0:1784948857.271336:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:2.0:1784948857.271336:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:3.0:1784948857.271338:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f13fb872. 02000000:00000001:1.0:1784948857.271343:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000010:2.0:1784948857.271344:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 02000000:00000010:1.0:1784948857.271350:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 00000100:00000001:2.0:1784948857.271357:0:337121:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1784948857.271360:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000010:3.0:1784948857.271361:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002e1eb6f6. 00000020:00000001:1.0:1784948857.271365:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948857.271373:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000200:2.0:1784948857.271373:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948857.271381:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948857.271385:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b84187a1. 02000000:00000001:1.0:1784948857.271393:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948857.271397:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948857.271399:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b3c18d2b. 00000100:00000200:2.0:1784948857.271403:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948857.271405:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948857.271409:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948857.271415:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948857.271425:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948857.271428:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.271429:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948857.271431:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948857.271435:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.271436:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.271442:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948857.271445:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948857.271447:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948857.271451:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948857.271455:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948857.271463:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=24 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948857.271468:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=24 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948857.271471:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948857.271473:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948857.271476:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948857.271482:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948857.271503:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.271508:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948857.271510:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948857.271513:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948857.271515:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948857.271520:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948857.271530:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948857.271533:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948857.271535:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948857.271539:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948857.271550:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289657600:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948857.271557:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.271560:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948857.271563:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c39699a8 x1871651289657600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948857.271573:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948857.271575:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948857.271578:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 02000000:00000001:2.0:1784948857.271587:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948857.271590:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948857.271592:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948857.271595:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948857.271597:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000595fa8f8. 02000000:00000001:2.0:1784948857.271600:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948857.271602:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948857.271604:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c39699a8. 00000100:00000001:2.0:1784948857.271608:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948857.271610:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.271612:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948857.271619:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948857.271620:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948857.271623:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948857.271625:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948857.312019:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305796651, at 4305796652 00000100:00000010:0.0:1784948857.312037:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000bbdc2de7. 00000100:00000040:1.0:1784948857.312041:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305796651, at 4305796652 00000100:00000040:2.0:1784948857.312041:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305796651, at 4305796652 00000100:00000040:3.0:1784948857.312049:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305796651, at 4305796652 00000100:00000010:2.0:1784948857.312063:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000004f2eb99f. 00000100:00000010:3.0:1784948857.312068:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:1.0:1784948857.312075:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005975f076. 00010000:00000001:2.0:1784948858.100043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948858.100052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948858.100059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948858.100061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948858.100225:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948858.100234:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948858.100239:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948858.292250:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948858.292270:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948858.292274:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948858.292282:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948858.292286:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948858.292292:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948858.292295:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948859.124047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948859.124061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948859.124086:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948859.124089:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948859.124238:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948859.124247:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948859.124253:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948859.316175:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948859.316191:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948859.316194:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948859.316200:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948859.316204:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948859.316208:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948859.316212:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948860.148187:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948860.148205:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948860.148219:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948860.148226:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948860.148230:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948860.148236:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948860.148241:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948860.148244:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948860.148246:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948860.148250:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948860.148357:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948860.148382:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948860.148388:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948860.340128:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948860.340147:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948860.340149:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948860.340155:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948860.340159:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948860.340163:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948860.340167:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948861.172055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948861.172066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948861.172076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948861.172078:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948861.172170:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948861.172178:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948861.172182:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948861.364043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948861.364062:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948861.364065:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948861.364071:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948861.364075:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948861.364079:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948861.364082:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948862.196191:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948862.196201:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948862.196209:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948862.196211:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948862.196321:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948862.196329:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948862.196333:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948862.388044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948862.388061:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948862.388063:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.388069:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948862.388072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948862.388076:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948862.388081:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.388163:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948862.388173:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948862.388185:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a612e3ff. 00000020:00000040:1.0:1784948862.388193:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948862.388199:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948862.388202:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948862.388209:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948862.388219:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000031deb17e. 00000100:00000001:1.0:1784948862.388228:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.388238:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948862.388253:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948862.388255:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.388262:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a612e3ff] to pc [ptlrpcd_00_03+3] req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948862.388302:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948862.388306:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948862.388311:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948862.388314:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948862.388318:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948862.388321:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000096f3921c. 00000020:00000040:1.0:1784948862.388324:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948862.388327:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948862.388329:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948862.388331:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948862.388334:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:1.0:1784948862.388338:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.388347:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948862.388357:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948862.388359:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.388365:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096f3921c] to pc [ptlrpcd_00_00+0] req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948862.388365:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948862.388374:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948862.388380:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948862.388385:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948862.388394:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948862.388402:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.388411:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948862.388427:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784948862.388428:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11139) 00000100:00000001:3.0:1784948862.388432:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948862.388440:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948862.388444:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948862.388448:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.388454:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784948862.388455:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289657728:0@lo:400: 00000100:00100000:1.0:1784948862.388461:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:3.0:1784948862.388464:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948862.388467:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948862.388470:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.388474:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948862.388475:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948862.388482:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784948862.388483:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.388490:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948862.388492:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948862.388497:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948862.388502:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1784948862.388503:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000898cc47a. 00000100:00000001:1.0:1784948862.388505:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1784948862.388513:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.388518:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948862.388525:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fde3a35c. 00000100:00000001:1.0:1784948862.388537:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948862.388541:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948862.388541:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 02000000:00000001:1.0:1784948862.388552:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948862.388556:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784948862.388558:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657728, portal 25 02000000:00000001:1.0:1784948862.388561:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948862.388566:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1784948862.388570:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000096f3921c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289657856:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948862.388573:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549158016 : -115544160393600 : ffff96e9c75d2680) 00000100:00000001:1.0:1784948862.388583:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948862.388588:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784948862.388591:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948862.388593:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948862.388601:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948862.388610:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 00000100:00000001:3.0:1784948862.388612:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784948862.388616:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948862.388624:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948862.388630:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000100:00000200:3.0:1784948862.388633:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289657728, offset 0 00000400:00000200:3.0:1784948862.388642:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:1.0:1784948862.388644:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fab125ff. 00000400:00000010:1.0:1784948862.388654:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000400:00000200:3.0:1784948862.388662:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:1.0:1784948862.388667:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657856, portal 10 00000100:00000001:1.0:1784948862.388673:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948862.388675:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549153536 : -115544160398080 : ffff96e9c75d1500) 00000400:00000200:3.0:1784948862.388684:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437180 00000100:00000040:1.0:1784948862.388687:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948862.388703:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7be1 [2] + 0 00000100:00000001:1.0:1784948862.388717:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:3.0:1784948862.388720:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e2d054c8. 00000100:00000040:1.0:1784948862.388728:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948862.388733:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bcd2638a 00000400:00000010:1.0:1784948862.388735:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000100:00000200:1.0:1784948862.388745:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289657856, offset 0 00000400:00000200:1.0:1784948862.388753:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948862.388753:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948862.388766:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948862.388774:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948862.388781:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948862.388787:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948862.388792:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948862.388796:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948862.388801:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3335:3335:236:4294967295] 192.168.202.137@tcp LPNI seq info [2152:2152:8:4294967295] 00000400:00000200:1.0:1784948862.388816:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948862.388822:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948862.388828:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000400:00000200:3.0:1784948862.388831:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bcd2638a 00000800:00000200:1.0:1784948862.388839:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:3.0:1784948862.388841:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bcd2638a. 00000100:00000001:3.0:1784948862.388855:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:1.0:1784948862.388861:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000200:3.0:1784948862.388862:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:1.0:1784948862.388870:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:3.0:1784948862.388876:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000014ab6f86 x1871651289657728 msgsize 224 00000100:00100000:3.0:1784948862.388890:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:1.0:1784948862.388893:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948862.388896:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:1.0:1784948862.388944:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2067 usecs (3 jiffies) 00000100:00000001:1.0:1784948862.388951:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305801732 00000100:00000010:1.0:1784948862.388959:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:1.0:1784948862.388968:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784948862.388983:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305801732 00000100:00000010:1.0:1784948862.388987:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:3.0:1784948862.388990:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948862.388992:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948862.389007:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305801732 00000400:00000200:3.0:1784948862.389011:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000100:00000010:1.0:1784948862.389012:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000400:00000010:3.0:1784948862.389018:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:1.0:1784948862.389022:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948862.389028:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948862.389029:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948862.389033:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305801732 00000100:00000200:3.0:1784948862.389042:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948862.389046:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657728 00000100:00000010:1.0:1784948862.389049:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 02000000:00000001:2.0:1784948862.389065:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948862.389066:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948862.389074:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948862.389079:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948862.389081:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948862.389090:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948862.389118:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.389119:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.389137:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.389153:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948862.389156:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948862.389158:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948862.389177:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784948862.389181:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657728 00000020:00000001:2.0:1784948862.389190:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948862.389196:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948862.389198:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000040:2.0:1784948862.389198:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784948862.389207:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305801771 00000020:00000001:2.0:1784948862.389213:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000010:0.0:1784948862.389214:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000100:00000001:3.0:1784948862.389223:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305801732 00000020:00000040:2.0:1784948862.389225:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:0.0:1784948862.389227:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784948862.389243:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948862.389245:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305801771 00000400:00000200:0.0:1784948862.389251:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784948862.389265:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948862.389266:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305801732 00000400:00000200:0.0:1784948862.389268:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000400:00000010:0.0:1784948862.389278:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000100:00000001:3.0:1784948862.389285:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305801771 00000100:00000001:0.0:1784948862.389292:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948862.389295:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305801732 00000100:00000001:2.0:1784948862.389297:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:0.0:1784948862.389305:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948862.389308:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305801771 00000020:00000010:2.0:1784948862.389314:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000cfb525fd. 00000100:00000001:3.0:1784948862.389320:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305801732 00000100:00000001:2.0:1784948862.389329:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948862.389334:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948862.389351:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000cfb525fd. 00000100:00000001:3.0:1784948862.389353:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948862.389360:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948862.389372:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948862.389373:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948862.389380:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000041892d66. 00000100:00000001:3.0:1784948862.389384:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948862.389403:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948862.389405:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948862.389407:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003562bf29. 00000100:00000001:3.0:1784948862.389410:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948862.389411:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948862.389424:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1784948862.389427:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000001:3.0:1784948862.389429:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:0.0:1784948862.389430:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948862.389433:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948862.389442:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948862.389450:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948862.389457:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784948862.389467:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000001:0.0:1784948862.389469:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948862.389487:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948862.389501:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784948862.389502:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948862.389518:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948862.389527:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437200 00000100:00000001:2.0:1784948862.389538:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948862.389540:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 157168 00000800:00000001:0.0:1784948862.389557:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948862.389566:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948862.389569:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948862.389574:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948862.389578:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.389581:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948862.389588:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784948862.389595:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000075b1f5f6. 00000100:00000040:0.0:1784948862.389600:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000075b1f5f6 x1871651289657856 msgsize 224 00000100:00000001:2.0:1784948862.389616:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:0.0:1784948862.389618:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784948862.389630:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948862.389648:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 721 00000100:00000001:3.0:1784948862.389651:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948862.389653:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948862.389658:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:2.0:1784948862.389659:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000800:00000001:0.0:1784948862.389667:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948862.389668:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948862.389671:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782155856 : -115538927395760 : ffff96eaff465c50) 00000800:00000001:0.0:1784948862.389676:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948862.389677:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948862.389690:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000014ab6f86 x1871651289657728/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:523/0 lens 224/0 e 0 to 0 dl 1784948873 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948862.389712:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948862.389720:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000cfb525fd. 00000100:00000001:2.0:1784948862.389727:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948862.389732:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000cfb525fd. 00000100:00100000:2.0:1784948862.389744:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000014ab6f86 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289657728:12345-0@lo:400: 00000100:00000200:2.0:1784948862.389752:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657728 00000020:00000001:2.0:1784948862.389759:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948862.389766:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948862.389771:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.389772:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:2.0:1784948862.389778:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00100000:1.0:1784948862.389780:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657856 02000000:00000001:1.0:1784948862.389787:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:2.0:1784948862.389788:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784948862.389791:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948862.389797:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948862.389803:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 02000000:00000001:1.0:1784948862.389805:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948862.389811:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948862.389814:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657856 00000020:00000001:1.0:1784948862.389825:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:2.0:1784948862.389826:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000040:1.0:1784948862.389835:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784948862.389836:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948862.389845:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:2.0:1784948862.389849:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948862.389851:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948862.389867:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948862.389868:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000001:1.0:1784948862.389879:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948862.389882:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948862.389887:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:2.0:1784948862.389892:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:1.0:1784948862.389896:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948862.389938:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948862.389944:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000010:1.0:1784948862.389955:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dd0652fb. 02000000:00000010:2.0:1784948862.389960:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000004dc9167. 00000020:00000010:1.0:1784948862.389963:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f6157664. 02000000:00000001:2.0:1784948862.389974:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948862.389977:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948862.389989:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.389997:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:2.0:1784948862.389999:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.390002:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948862.390007:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00010000:00000040:2.0:1784948862.390011:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657728 00000100:00000001:1.0:1784948862.390012:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948862.390030:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784948862.390035:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948862.390040:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948862.390045:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:2.0:1784948862.390045:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000014ab6f86 x1871651289657728/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:523/0 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948862.390071:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948862.390072:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948862.390081:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.390088:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00001000:2.0:1784948862.390091:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=34 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948862.390124:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948862.390125:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948862.390137:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00100000:1.0:1784948862.390145:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 697 00000100:00000001:3.0:1784948862.390151:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948862.390152:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000040:1.0:1784948862.390156:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948862.390158:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948862.390170:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.390173:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558301184 : -115544151250432 : ffff96e9c7e8aa00) 02000000:00000001:2.0:1784948862.390175:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784948862.390177:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948862.390186:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.390192:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000075b1f5f6 x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:523/0 lens 224/0 e 0 to 0 dl 1784948873 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948862.390200:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948862.390228:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784948862.390240:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:2.0:1784948862.390244:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000020:00000010:1.0:1784948862.390245:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948862.390263:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784948862.390270:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289657728, offset 224 00000020:00000010:1.0:1784948862.390271:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00100000:1.0:1784948862.390285:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000075b1f5f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289657856:12345-192.168.202.137@tcp:400: 00000400:00000200:2.0:1784948862.390300:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000200:1.0:1784948862.390301:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657856 00000020:00000001:1.0:1784948862.390309:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948862.390315:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948862.390318:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948862.390324:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948862.390330:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:2.0:1784948862.390335:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:1.0:1784948862.390338:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948862.390347:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948862.390352:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948862.390354:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948862.390358:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948862.390370:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:2.0:1784948862.390372:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437180 00000100:00000001:1.0:1784948862.390384:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948862.390388:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948862.390395:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000065e77d68. 00000400:00000200:2.0:1784948862.390396:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x85ed [1] + 224 02000000:00000001:1.0:1784948862.390404:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.390418:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948862.390422:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948862.390425:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657856 00000400:00000200:2.0:1784948862.390427:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1784948862.390431:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948862.390445:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000075b1f5f6 x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:523/0 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948862.390449:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00010000:00000001:1.0:1784948862.390475:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784948862.390477:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:1.0:1784948862.390480:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948862.390486:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=29 v=5 (1 1 1 1) 00000100:00000200:2.0:1784948862.390493:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948862.390496:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948862.390507:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948862.390519:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948862.390536:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948862.390541:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.390557:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948862.390564:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948862.390567:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948862.390579:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b039801a. 00000100:00000200:1.0:1784948862.390591:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289657856, offset 224 00000400:00000200:1.0:1784948862.390612:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948862.390627:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948862.390630:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948862.390638:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3336:3336:236:4294967295] 192.168.202.137@tcp LPNI seq info [2153:2153:8:4294967295] 00000400:00000200:1.0:1784948862.390653:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948862.390659:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948862.390663:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000400:00000200:2.0:1784948862.390670:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000800:00000200:1.0:1784948862.390671:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948862.390689:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948862.390695:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:2.0:1784948862.390696:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000100:00000001:2.0:1784948862.390711:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948862.390714:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948862.390722:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948862.390731:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.390732:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948862.390747:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784948862.390748:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948862.390753:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:1.0:1784948862.390755:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948862.390758:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1784948862.390770:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00010000:00000001:2.0:1784948862.390773:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:3.0:1784948862.390777:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1784948862.390780:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.390782:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000075b1f5f6 x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:523/0 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784948862.390783:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948862.390795:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fde3a35c. 00000020:00000001:2.0:1784948862.390798:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:3.0:1784948862.390820:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000100:00000040:2.0:1784948862.390823:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000014ab6f86 x1871651289657728/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:523/0 lens 224/224 e 0 to 0 dl 1784948873 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948862.390828:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00100000:1.0:1784948862.390833:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000075b1f5f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289657856:12345-192.168.202.137@tcp:400: Request processed in 576us (1216us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948862.390841:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948862.390860:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948862.390893:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 697 00000100:00000200:3.0:1784948862.390959:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948862.390975:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948862.390990:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948862.391000:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948862.391015:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f6157664. 00000800:00000200:0.0:1784948862.391016:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948862.391018:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948862.391029:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948862.391031:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000100:00100000:2.0:1784948862.391031:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000014ab6f86 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289657728:12345-0@lo:400: Request processed in 1303us (2139us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948862.391038:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:1.0:1784948862.391039:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000400:00000200:0.0:1784948862.391051:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784948862.391052:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:1.0:1784948862.391053:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dd0652fb. 00000100:00000001:3.0:1784948862.391059:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:1.0:1784948862.391074:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948862.391075:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948862.391086:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948862.391087:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784948862.391089:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948862.391109:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 721 00000100:00000040:2.0:1784948862.391121:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00001000:3.0:1784948862.391129:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=34 v=5 (5 5 5 5) 00000400:00000200:0.0:1784948862.391132:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b039801a 00000100:00000001:2.0:1784948862.391133:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00001000:3.0:1784948862.391139:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=34 v=5 (1 1 1 1) 00000400:00000010:0.0:1784948862.391142:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b039801a. 00000100:00000001:2.0:1784948862.391145:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:1.0:1784948862.391158:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948862.391162:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:2.0:1784948862.391167:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003562bf29. 00000100:00000001:0.0:1784948862.391169:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948862.391169:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948862.391174:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.391177:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00010000:00000001:3.0:1784948862.391189:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000800:00000001:1.0:1784948862.391191:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784948862.391200:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00010000:00000001:3.0:1784948862.391201:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948862.391202:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948862.391221:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948862.391223:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000041892d66. 00000400:00000200:1.0:1784948862.391229:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:3.0:1784948862.391239:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784948862.391250:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948862.391256:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948862.391262:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437200 02000000:00000001:2.0:1784948862.391270:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948862.391270:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:1.0:1784948862.391285:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x85f5 [1] + 224 02000000:00000010:2.0:1784948862.391289:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000004dc9167. 00000100:00000001:3.0:1784948862.391290:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948862.391295:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948862.391315:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948862.391315:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000040:3.0:1784948862.391316:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948862.391329:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948862.391329:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948862.391360:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:3.0:1784948862.391361:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289657728:0@lo:400: 00000400:00000200:1.0:1784948862.391362:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948862.391384:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000010:2.0:1784948862.391385:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001a3c4d34. 00000100:00000001:3.0:1784948862.391385:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948862.391399:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948862.391404:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:2.0:1784948862.391410:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f7c52f62. 00000100:00000040:3.0:1784948862.391419:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a612e3ff x1871651289657728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948862.391437:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948862.391438:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948862.391446:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948862.391453:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000898cc47a. 02000000:00000001:3.0:1784948862.391467:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948862.391471:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948862.391473:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784948862.391478:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:1.0:1784948862.391480:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:3.0:1784948862.391481:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000031deb17e. 02000000:00000001:3.0:1784948862.391489:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948862.391492:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000001:1.0:1784948862.391493:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000010:3.0:1784948862.391494:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a612e3ff. 00000800:00000001:1.0:1784948862.391502:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948862.391505:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948862.391512:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948862.391516:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948862.391523:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948862.391526:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948862.391532:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948862.391536:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948862.391537:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948862.391543:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948862.391551:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948862.391553:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948862.391557:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fab125ff. 00000400:00000200:0.0:1784948862.391565:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:0.0:1784948862.391568:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000100:00000001:0.0:1784948862.391573:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948862.391580:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948862.391593:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948862.391603:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948862.391605:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.391608:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948862.391610:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948862.391612:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948862.391614:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948862.391617:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948862.391624:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=29 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948862.391629:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=29 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948862.391631:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948862.391633:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948862.391636:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948862.391640:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948862.391657:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948862.391661:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948862.391663:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948862.391665:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948862.391667:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948862.391670:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948862.391678:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948862.391681:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948862.391682:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948862.391686:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948862.391694:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000096f3921c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289657856:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948862.391699:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.391701:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948862.391704:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000096f3921c x1871651289657856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948862.391712:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948862.391714:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948862.391717:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 02000000:00000001:0.0:1784948862.391724:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948862.391725:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948862.391727:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948862.391730:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948862.391732:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 02000000:00000001:0.0:1784948862.391735:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948862.391737:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948862.391739:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000096f3921c. 00000100:00000001:0.0:1784948862.391742:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948862.391743:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.391745:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948862.391751:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948862.391753:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948862.391755:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948862.391758:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948862.432011:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305801771, at 4305801772 00000100:00000010:1.0:1784948862.432026:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:0.0:1784948862.432069:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305801771, at 4305801772 00000100:00000040:2.0:1784948862.432086:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305801771, at 4305801772 00000100:00000010:0.0:1784948862.432121:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:2.0:1784948862.432133:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000040:3.0:1784948862.432175:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305801771, at 4305801772 00000100:00000010:3.0:1784948862.432186:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784948863.220074:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948863.220119:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948863.220131:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948863.220133:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948863.220229:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948863.220237:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948863.220242:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948863.412034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948863.412051:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948863.412054:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948863.412060:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948863.412064:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948863.412068:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948863.412071:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948864.244069:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948864.244104:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948864.244139:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948864.244141:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948864.244272:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948864.244282:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948864.244287:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948864.436119:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948864.436153:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948864.436155:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948864.436162:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948864.436166:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948864.436171:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948864.436173:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948865.268045:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948865.268054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948865.268063:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948865.268065:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948865.268188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948865.268194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948865.268199:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948865.460094:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948865.460118:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948865.460121:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948865.460130:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948865.460135:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948865.460141:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948865.460144:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1784948865.972184:0:343954:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.25:988 00000400:00000100:1.0:1784948865.972213:0:343958:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.14:988 00000400:00000100:2.0:1784948865.972255:0:343654:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.6:988 00000400:00000100:0.0:1784948865.972341:0:343914:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.12:988 00000400:00000100:3.0:1784948865.972359:0:343954:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.25@tcp at host 111.222.142.25:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784948865.972372:0:343958:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.14@tcp at host 111.222.142.14:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948865.972390:0:343954:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.25@tcp: down 00000400:00000100:2.0:1784948865.972401:0:343654:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.6@tcp at host 111.222.142.6:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948865.972405:0:343958:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.14@tcp: down 00000400:00000200:1.0:1784948865.972423:0:343958:0:(router.c:1888:lnet_notify()) Old news 00000800:00000200:3.0:1784948865.972430:0:343954:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.142.25@tcp down 00000400:00000100:0.0:1784948865.972437:0:343914:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.12@tcp at host 111.222.142.12:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948865.972445:0:343654:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.6@tcp: down 00000800:00000100:1.0:1784948865.972452:0:343958:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000400:00000200:2.0:1784948865.972462:0:343654:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:1.0:1784948865.972464:0:343958:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000036e798e7. 00000400:00000200:0.0:1784948865.972480:0:343914:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.12@tcp: down 00000400:00000200:1.0:1784948865.972485:0:343958:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948865.972491:0:343914:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784948865.972513:0:343958:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000800:00000100:0.0:1784948865.972514:0:343914:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:0.0:1784948865.972531:0:343914:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000762a0fe3. 00000400:00000200:1.0:1784948865.972543:0:343958:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11139 health :900 00000400:00000200:0.0:1784948865.972551:0:343914:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948865.972558:0:343958:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948865.972575:0:343914:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948865.972575:0:343958:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:1.0:1784948865.972597:0:343958:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:0.0:1784948865.972624:0:343914:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948865.972637:0:343914:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:0.0:1784948865.972647:0:343914:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000100:3.0:1784948865.972708:0:343755:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.4:988 00000400:00000100:0.0:1784948865.972714:0:343955:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.10:988 00000400:00000200:2.0:1784948865.972755:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948865.972776:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:2.0:1784948865.972790:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948865.972795:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948865.972800:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000010:2.0:1784948865.972805:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000400:00000200:2.0:1784948865.972817:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000100:0.0:1784948865.972823:0:343955:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.10@tcp at host 111.222.142.10:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784948865.972838:0:343755:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.4@tcp at host 111.222.142.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948865.972842:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:0.0:1784948865.972852:0:343955:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.10@tcp: down 00000400:00000100:1.0:1784948865.972855:0:343953:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.8:988 00000400:00000010:2.0:1784948865.972859:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b1cbf66f. 00000400:00000200:3.0:1784948865.972871:0:343755:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.4@tcp: down 00000400:00000200:0.0:1784948865.972883:0:343955:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948865.972884:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:3.0:1784948865.972898:0:343755:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948865.972900:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000800:00000100:3.0:1784948865.972967:0:343755:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.4@tcp 00000800:00000010:3.0:1784948865.972972:0:343755:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009839174d. 00000400:00000200:3.0:1784948865.972979:0:343755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000100:0.0:1784948865.972994:0:343915:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.2:988 00000400:00000200:2.0:1784948865.972997:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00000400:00000200:3.0:1784948865.973007:0:343755:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948865.973016:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000200:3.0:1784948865.973017:0:343755:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000001:2.0:1784948865.973033:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:3.0:1784948865.973039:0:343755:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000001:2.0:1784948865.973043:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948865.973051:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784948865.973056:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000400:00000200:3.0:1784948865.973059:0:343755:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000100:0.0:1784948865.973085:0:343915:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.2@tcp at host 111.222.142.2:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948865.973099:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948865.973109:0:343915:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.2@tcp: down 00000400:00000200:2.0:1784948865.973120:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:2.0:1784948865.973128:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000063f28b6f. 00000400:00000200:2.0:1784948865.973144:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000400:00000200:0.0:1784948865.973146:0:343915:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948865.973160:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000400:00000200:2.0:1784948865.973175:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000400:00000100:1.0:1784948865.973186:0:343953:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.8@tcp at host 111.222.142.8:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948865.973194:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:2.0:1784948865.973201:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948865.973207:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784948865.973208:0:343953:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.8@tcp: down 00000400:00000200:2.0:1784948865.973215:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000200:1.0:1784948865.973219:0:343953:0:(router.c:1888:lnet_notify()) Old news 00000400:00000010:2.0:1784948865.973220:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000400:00000200:2.0:1784948865.973230:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000100:1.0:1784948865.973234:0:343953:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.8@tcp 00000400:00000200:2.0:1784948865.973241:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000800:00000010:1.0:1784948865.973243:0:343953:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000016632059. 00000400:00000010:2.0:1784948865.973247:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000053ab16eb. 00000400:00000200:1.0:1784948865.973253:0:343953:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948865.973260:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-110 00000400:00000200:1.0:1784948865.973269:0:343953:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948865.973275:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -110 00000400:00000200:1.0:1784948865.973280:0:343953:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948865.973283:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:1.0:1784948865.973291:0:343953:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:1.0:1784948865.973299:0:343953:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:2.0:1784948865.973337:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948865.973346:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:2.0:1784948865.973350:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948865.973352:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948865.973353:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:2.0:1784948865.973356:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000400:00000200:2.0:1784948865.973360:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948865.973363:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:2.0:1784948865.973366:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000400:00000200:2.0:1784948865.973371:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-110 00000400:00000200:2.0:1784948865.973373:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -110 00000400:00000200:2.0:1784948865.973377:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00010000:00000001:2.0:1784948866.292104:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948866.292113:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948866.292121:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948866.292123:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948866.292201:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948866.292210:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948866.292215:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948866.484040:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948866.484091:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948866.484094:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948866.484102:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948866.484107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948866.484112:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948866.484116:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948867.316098:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948867.316114:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948867.316125:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948867.316127:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948867.316262:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948867.316270:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948867.316275:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1784948867.508112:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948867.508131:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948867.508136:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000010:3.0:1784948867.508144:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fa1e143a. 00000100:00000001:0.0:1784948867.508151:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948867.508155:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784948867.508157:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948867.508161:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948867.508166:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948867.508166:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948867.508174:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784948867.508175:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948867.508180:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948867.508185:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948867.508201:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:3.0:1784948867.508213:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948867.508228:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948867.508246:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948867.508249:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948867.508257:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fa1e143a] to pc [ptlrpcd_00_01+1] req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948867.508303:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948867.508308:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:0.0:1784948867.508312:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784948867.508315:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1784948867.508316:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948867.508321:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784948867.508324:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1784948867.508330:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948867.508333:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948867.508340:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1784948867.508340:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005dccb977. 00000100:00000001:0.0:1784948867.508344:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948867.508347:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948867.508349:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948867.508349:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948867.508373:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948867.508374:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948867.508378:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948867.508381:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948867.508386:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948867.508388:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948867.508400:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fa1e143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289657984:0@lo:400: 02000000:00000010:3.0:1784948867.508401:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:0.0:1784948867.508411:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948867.508414:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948867.508425:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948867.508431:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948867.508432:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1784948867.508438:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948867.508451:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 02000000:00000001:0.0:1784948867.508461:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948867.508461:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948867.508468:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948867.508474:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bb684072. 00000100:00000040:3.0:1784948867.508478:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005dccb977] to pc [ptlrpcd_00_02+2] req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948867.508489:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000089c8262f. 00000100:00000200:0.0:1784948867.508506:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289657984, portal 25 00000100:00000001:0.0:1784948867.508515:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948867.508519:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482520704 : -115544227030912 : ffff96e9c3645880) 00000100:00000040:0.0:1784948867.508533:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948867.508545:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948867.508549:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948867.508552:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000100:00000200:0.0:1784948867.508555:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289657984, offset 0 00000400:00000200:0.0:1784948867.508564:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948867.508579:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948867.508610:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11144) 00000400:00000200:0.0:1784948867.508616:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948867.508633:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437280 00000400:00000200:0.0:1784948867.508643:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7c09 [2] + 0 00000400:00000010:0.0:1784948867.508648:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000100:00000001:1.0:1784948867.508652:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948867.508656:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000400:00000200:0.0:1784948867.508657:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000000575a8ab 00000100:00000001:1.0:1784948867.508659:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.508666:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948867.508667:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948867.508668:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948867.508671:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948867.508674:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948867.508675:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948867.508680:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948867.508681:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948867.508683:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784948867.508690:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000575a8ab 00000100:00000040:1.0:1784948867.508691:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948867.508695:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000000575a8ab. 00000100:00000001:0.0:1784948867.508704:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948867.508705:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948867.508709:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948867.508710:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:1.0:1784948867.508712:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948867.508715:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000000407c50 x1871651289657984 msgsize 224 02000000:00000001:1.0:1784948867.508716:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948867.508721:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948867.508727:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1784948867.508729:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289658112:192.168.202.137@tcp:400: 00000100:00000040:0.0:1784948867.508731:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948867.508737:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948867.508740:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948867.508741:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948867.508743:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948867.508747:0:337120:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:1.0:1784948867.508747:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000378c7327. 02000000:00000001:1.0:1784948867.508752:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948867.508757:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d6b8e52f. 00000400:00000200:0.0:1784948867.508759:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:0.0:1784948867.508762:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00000001:0.0:1784948867.508766:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:1.0:1784948867.508771:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000200:0.0:1784948867.508773:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948867.508777:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289658112, portal 10 00000100:00000001:1.0:1784948867.508783:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948867.508786:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482519808 : -115544227031808 : ffff96e9c3645500) 00000100:00000001:0.0:1784948867.508791:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948867.508797:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948867.508800:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948867.508800:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948867.508807:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289657984 02000000:00000001:3.0:1784948867.508814:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948867.508820:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948867.508825:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948867.508831:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948867.508834:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948867.508835:0:337120:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1784948867.508835:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948867.508841:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948867.508843:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948867.508845:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289657984 00000400:00000010:1.0:1784948867.508846:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000100:00000001:0.0:1784948867.508849:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305806890 00000100:00000200:1.0:1784948867.508852:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289658112, offset 0 00000020:00000001:3.0:1784948867.508854:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948867.508859:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:0.0:1784948867.508860:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000400:00000200:1.0:1784948867.508864:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784948867.508867:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948867.508872:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000040:3.0:1784948867.508874:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:1.0:1784948867.508879:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:3.0:1784948867.508882:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948867.508885:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305806890 00000400:00000200:1.0:1784948867.508887:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784948867.508889:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:0.0:1784948867.508892:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000400:00000200:1.0:1784948867.508897:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784948867.508900:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948867.508900:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948867.508931:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305806891 00000100:00000010:0.0:1784948867.508935:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000400:00000200:1.0:1784948867.508937:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948867.508941:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000400:00000200:1.0:1784948867.508944:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:3.0:1784948867.508945:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec8cbdfe. 00000100:00000001:0.0:1784948867.508952:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305806891 00000100:00000001:3.0:1784948867.508953:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948867.508958:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3337:3337:240:4294967295] 192.168.202.137@tcp LPNI seq info [2154:2154:8:4294967295] 00000100:00000010:0.0:1784948867.508961:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000020:00000010:3.0:1784948867.508963:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec8cbdfe. 00000100:00000001:0.0:1784948867.508968:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948867.508975:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948867.508977:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000898cc47a. 00000400:00000200:1.0:1784948867.508979:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948867.508980:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948867.508986:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948867.508989:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000eefa0203. 00000100:00000001:0.0:1784948867.508990:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948867.508994:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948867.509000:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:3.0:1784948867.509001:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000800:00000010:1.0:1784948867.509002:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000adf08f9b. 00000100:00000001:0.0:1784948867.509003:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948867.509010:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948867.509012:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948867.509014:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948867.509018:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948867.509020:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948867.509022:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948867.509029:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000800:00000200:1.0:1784948867.509033:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784948867.509034:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948867.509039:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948867.509040:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948867.509074:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948867.509077:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948867.509084:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2071 usecs (3 jiffies) 00000100:00000001:1.0:1784948867.509086:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305806852 00000100:00000001:1.0:1784948867.509089:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305806890 00000100:00000001:1.0:1784948867.509091:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305806852 00000100:00000001:1.0:1784948867.509095:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305806890 00000100:00000001:3.0:1784948867.509095:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.509100:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305806852 00000100:00000001:1.0:1784948867.509104:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305806891 00000100:00000001:1.0:1784948867.509105:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305806852 00000100:00000001:1.0:1784948867.509107:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305806891 00000100:00000001:1.0:1784948867.509109:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948867.509111:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948867.509114:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948867.509135:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948867.509135:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948867.509142:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948867.509154:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 722 00000100:00000040:3.0:1784948867.509162:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948867.509167:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782148688 : -115538927402928 : ffff96eaff464050) 00000100:00000040:3.0:1784948867.509176:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000000407c50 x1871651289657984/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:528/0 lens 224/0 e 0 to 0 dl 1784948878 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948867.509192:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948867.509198:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec8cbdfe. 00000100:00000001:3.0:1784948867.509201:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948867.509205:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec8cbdfe. 00000100:00100000:3.0:1784948867.509213:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000000407c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289657984:12345-0@lo:400: 00000100:00000200:3.0:1784948867.509220:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289657984 00000020:00000001:3.0:1784948867.509226:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948867.509232:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948867.509236:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948867.509241:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948867.509246:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:0.0:1784948867.509251:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:3.0:1784948867.509253:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000010:0.0:1784948867.509259:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000adf08f9b. 00000020:00000001:3.0:1784948867.509259:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948867.509268:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:0.0:1784948867.509269:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784948867.509273:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:1.0:1784948867.509278:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784948867.509279:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948867.509280:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784948867.509289:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:0.0:1784948867.509291:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:0.0:1784948867.509296:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000100:00000001:3.0:1784948867.509315:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948867.509319:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:1.0:1784948867.509319:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948867.509322:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:1.0:1784948867.509327:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784948867.509332:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784948867.509334:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008d04782c. 00000800:00000001:1.0:1784948867.509336:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948867.509346:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948867.509358:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948867.509361:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948867.509364:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1784948867.509368:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948867.509374:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:3.0:1784948867.509378:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289657984 00000400:00000200:1.0:1784948867.509388:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437300 00010000:00000001:3.0:1784948867.509392:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948867.509403:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784948867.509406:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000000407c50 x1871651289657984/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:528/0 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948867.509411:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784948867.509412:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 157392 00000800:00000001:1.0:1784948867.509432:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:3.0:1784948867.509441:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000800:00000001:1.0:1784948867.509444:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784948867.509447:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948867.509451:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00001000:3.0:1784948867.509458:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=39 v=5 (1 1 1 1) 00000400:00000200:1.0:1784948867.509459:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948867.509470:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948867.509474:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1784948867.509475:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948867.509486:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ac3cb588. 00000100:00000040:3.0:1784948867.509489:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:1.0:1784948867.509493:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000ac3cb588 x1871651289658112 msgsize 224 00000100:00000001:3.0:1784948867.509500:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00100000:1.0:1784948867.509507:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:3.0:1784948867.509514:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948867.509520:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948867.509527:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948867.509533:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:3.0:1784948867.509535:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948867.509545:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000200:3.0:1784948867.509553:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289657984, offset 224 00000800:00000001:1.0:1784948867.509554:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948867.509564:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948867.509570:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948867.509591:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784948867.509598:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948867.509604:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289658112 00000400:00000200:3.0:1784948867.509610:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437280 02000000:00000001:2.0:1784948867.509611:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948867.509616:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948867.509621:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948867.509627:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948867.509633:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8609 [1] + 224 00000100:00100000:2.0:1784948867.509634:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289658112 00000020:00000001:2.0:1784948867.509640:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948867.509646:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784948867.509666:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784948867.509668:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1784948867.509669:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784948867.509676:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784948867.509680:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948867.509687:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1784948867.509691:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:3.0:1784948867.509693:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948867.509702:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948867.509710:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:2.0:1784948867.509716:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948867.509718:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948867.509720:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000020:00000010:2.0:1784948867.509730:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000041892d66. 00000020:00000010:2.0:1784948867.509739:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000055acc4d8. 00000020:00000010:2.0:1784948867.509746:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000100:00000040:2.0:1784948867.509754:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948867.509756:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948867.509758:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948867.509759:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948867.509767:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948867.509774:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000100:00000001:2.0:1784948867.509775:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948867.509782:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000010:3.0:1784948867.509783:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00000001:2.0:1784948867.509787:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948867.509792:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948867.509796:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948867.509800:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948867.509813:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948867.509816:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948867.509823:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 698 00000100:00000001:0.0:1784948867.509827:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:2.0:1784948867.509827:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948867.509833:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948867.509834:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549151744 : -115544160399872 : ffff96e9c75d0e00) 00000100:00000001:3.0:1784948867.509836:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948867.509838:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948867.509846:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948867.509846:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ac3cb588 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:528/0 lens 224/0 e 0 to 0 dl 1784948878 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948867.509848:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784948867.509850:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948867.509857:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1784948867.509860:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948867.509866:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948867.509867:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784948867.509870:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000001:1.0:1784948867.509872:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:2.0:1784948867.509876:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000020:00000001:3.0:1784948867.509876:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:1.0:1784948867.509877:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bb684072. 00000100:00000001:2.0:1784948867.509885:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948867.509888:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000100:00000040:3.0:1784948867.509889:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000000407c50 x1871651289657984/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:528/0 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948867.509891:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000400:00000010:1.0:1784948867.509894:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000100:00000001:1.0:1784948867.509936:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784948867.509957:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ac3cb588 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289658112:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948867.509966:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948867.509978:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289658112 00000100:00100000:3.0:1784948867.509984:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000000407c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289657984:12345-0@lo:400: Request processed in 783us (1258us total) trans 0 rc 0/0 00000020:00000001:2.0:1784948867.509987:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948867.509995:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948867.510001:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948867.510012:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948867.510012:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948867.510019:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784948867.510029:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948867.510035:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948867.510041:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 722 00000020:00000001:2.0:1784948867.510044:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948867.510046:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1784948867.510069:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948867.510073:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948867.510078:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948867.510080:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948867.510080:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948867.510088:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948867.510089:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948867.510091:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948867.510112:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784948867.510114:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:1.0:1784948867.510116:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000010:2.0:1784948867.510119:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000344a00bc. 00000100:00000001:1.0:1784948867.510120:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:3.0:1784948867.510124:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000eefa0203. 00000100:00000001:1.0:1784948867.510125:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948867.510125:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948867.510134:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948867.510136:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948867.510139:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000100:00001000:1.0:1784948867.510142:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=39 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948867.510149:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=39 v=5 (1 1 1 1) 00000020:00000010:3.0:1784948867.510149:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000898cc47a. 00000020:00000001:2.0:1784948867.510151:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948867.510160:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289658112 00000100:00000001:1.0:1784948867.510161:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000040:3.0:1784948867.510164:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784948867.510165:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948867.510171:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000001:1.0:1784948867.510175:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784948867.510179:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948867.510179:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000200:2.0:1784948867.510180:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ac3cb588 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:528/0 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784948867.510186:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008d04782c. 00000100:00000001:1.0:1784948867.510189:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948867.510195:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000040:1.0:1784948867.510201:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948867.510202:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948867.510204:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948867.510209:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948867.510216:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:2.0:1784948867.510217:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=34 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948867.510225:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1784948867.510228:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fde3a35c. 00000100:00000040:2.0:1784948867.510229:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948867.510232:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948867.510236:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:1.0:1784948867.510240:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1784948867.510241:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:3.0:1784948867.510242:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000abb53582. 00000100:00000001:1.0:1784948867.510247:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948867.510247:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948867.510256:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948867.510259:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948867.510266:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948867.510274:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000100:00100000:1.0:1784948867.510277:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fa1e143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289657984:0@lo:400: 00000100:00000200:2.0:1784948867.510281:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289658112, offset 224 00000100:00000001:1.0:1784948867.510284:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.510288:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948867.510292:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fa1e143a x1871651289657984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948867.510293:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948867.510305:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:2.0:1784948867.510308:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:1.0:1784948867.510310:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948867.510332:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 00000400:00000200:2.0:1784948867.510337:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3338:3338:240:4294967295] 192.168.202.137@tcp LPNI seq info [2155:2155:8:4294967295] 02000000:00000001:1.0:1784948867.510340:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948867.510346:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948867.510350:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784948867.510359:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948867.510365:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 00000400:00000200:2.0:1784948867.510365:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:1.0:1784948867.510375:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:2.0:1784948867.510382:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784948867.510383:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948867.510388:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fa1e143a. 00000800:00000010:2.0:1784948867.510392:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003a8a98bc. 00000100:00000001:1.0:1784948867.510394:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948867.510400:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.510404:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948867.510405:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948867.510411:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948867.510416:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948867.510417:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784948867.510422:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1784948867.510425:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003a8a98bc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948867.510427:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948867.510449:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948867.510456:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948867.510459:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948867.510461:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948867.510464:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948867.510470:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ac3cb588 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:528/0 lens 224/224 e 0 to 0 dl 1784948878 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948867.510482:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ac3cb588 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289658112:12345-192.168.202.137@tcp:400: Request processed in 611us (988us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948867.510495:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 698 00000100:00000040:2.0:1784948867.510498:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948867.510501:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948867.510503:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948867.510506:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000055acc4d8. 00000020:00000010:2.0:1784948867.510513:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000020:00000010:2.0:1784948867.510517:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000041892d66. 00000020:00000040:2.0:1784948867.510523:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948867.510526:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948867.510601:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948867.510607:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000400:00000200:0.0:1784948867.510613:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948867.510618:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948867.510624:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948867.510631:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000800:00000001:2.0:1784948867.510632:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948867.510634:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000800:00000001:2.0:1784948867.510639:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948867.510642:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784948867.510644:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948867.510646:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948867.510671:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948867.510683:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437300 00000400:00000200:2.0:1784948867.510689:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8611 [1] + 224 00000800:00000001:2.0:1784948867.510694:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948867.510703:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948867.510705:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948867.510708:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948867.510713:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948867.510716:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948867.510725:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948867.510741:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948867.510748:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948867.510750:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.510780:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948867.510783:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948867.510787:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948867.510789:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948867.510792:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d6b8e52f. 00000400:00000200:1.0:1784948867.510796:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:1.0:1784948867.510798:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000100:00000001:1.0:1784948867.510800:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948867.510805:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948867.510814:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948867.510821:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948867.510822:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.510824:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948867.510826:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948867.510827:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948867.510828:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948867.510830:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948867.510835:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=34 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948867.510837:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=34 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948867.510839:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948867.510841:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948867.510843:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948867.510845:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948867.510852:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948867.510855:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948867.510857:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948867.510858:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948867.510859:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948867.510862:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948867.510868:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948867.510871:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948867.510872:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948867.510874:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948867.510888:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289658112:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948867.510893:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.510894:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948867.510896:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005dccb977 x1871651289658112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948867.510902:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948867.510921:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948867.510923:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000378c7327. 02000000:00000001:1.0:1784948867.510927:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948867.510928:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948867.510929:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948867.510931:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948867.510932:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:1.0:1784948867.510934:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948867.510936:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948867.510937:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005dccb977. 00000100:00000001:1.0:1784948867.510939:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948867.510941:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.510942:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948867.510945:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948867.510947:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948867.510949:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948867.510951:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948867.551025:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305806890, at 4305806891 00000100:00000010:0.0:1784948867.551049:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:2.0:1784948867.552004:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305806891, at 4305806892 00000100:00000010:2.0:1784948867.552018:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:1.0:1784948867.552085:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305806890, at 4305806892 00000100:00000040:3.0:1784948867.552091:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305806891, at 4305806892 00000100:00000010:1.0:1784948867.552101:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000010:3.0:1784948867.552106:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00010000:00000001:2.0:1784948868.340134:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948868.340147:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948868.340158:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948868.340159:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948868.340272:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948868.340279:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948868.340284:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948868.532102:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948868.532125:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948868.532128:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948868.532137:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948868.532142:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948868.532149:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948868.532153:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948869.364115:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948869.364129:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948869.364141:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948869.364144:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948869.364238:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948869.364246:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948869.364250:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948869.556132:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948869.556157:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948869.556160:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948869.556167:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948869.556172:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948869.556177:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948869.556181:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784948869.748192:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:3.0:1784948869.748207:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:3.0:1784948869.748228:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948869.748241:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948869.748254:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11142 health :900 00000400:00000200:3.0:1784948869.748265:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948869.748273:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:3.0:1784948869.748282:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:2.0:1784948869.748440:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948869.748453:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:2.0:1784948869.748464:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948869.748468:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948869.748472:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:2.0:1784948869.748476:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000400:00000200:2.0:1784948869.748482:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948869.748487:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:2.0:1784948869.748493:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009f047fe3. 00000400:00000200:2.0:1784948869.748500:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:2.0:1784948869.748503:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:2.0:1784948869.748508:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00010000:00000001:2.0:1784948870.388158:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948870.388169:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948870.388178:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948870.388182:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948870.388186:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948870.388190:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948870.388194:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948870.388197:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948870.388199:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948870.388201:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948870.388322:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948870.388328:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948870.388332:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948870.580143:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948870.580168:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948870.580172:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948870.580180:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948870.580185:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948870.580191:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948870.580195:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948870.772103:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948870.772123:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000048fd39a3. 00000400:00000010:1.0:1784948870.772141:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f7a7941e. 00000400:00000010:1.0:1784948870.772150:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000010:1.0:1784948870.772165:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bd6a7269. 00000400:00000200:1.0:1784948870.772171:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948870.772194:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948870.772202:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948870.772208:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3339:3339:241:4294967295] 192.168.202.137@tcp LPNI seq info [2156:2156:8:4294967295] 00000400:00000200:1.0:1784948870.772217:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948870.772226:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948870.772233:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948870.772240:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000800:00000200:1.0:1784948870.772250:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948870.772262:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948870.772268:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948870.772299:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11144 health :900 00000800:00000200:0.0:1784948870.772550:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948870.772558:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:0.0:1784948870.772572:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948870.772572:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948870.772585:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784948870.772594:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948870.772595:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:1.0:1784948870.772605:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948870.772606:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948870.772617:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948870.772630:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948870.772646:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948870.772659:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948870.772671:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948870.772676:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3340:3340:241:4294967295] 192.168.202.137@tcp LPNI seq info [2157:2157:8:4294967295] 00000400:00000200:1.0:1784948870.772686:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948870.772691:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948870.772695:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000adf08f9b. 00000800:00000200:1.0:1784948870.772700:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948870.772705:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948870.772707:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784948870.772832:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948870.772835:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948870.772846:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000adf08f9b. 00000400:00000200:1.0:1784948870.772855:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948870.772858:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948870.772862:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948870.772875:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948870.772882:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948870.772888:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948870.772898:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948870.772963:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948870.772978:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8625 00000800:00000001:0.0:1784948870.772985:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948870.772999:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948870.773001:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948870.773005:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948870.773012:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:0.0:1784948870.773015:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000400:00000200:0.0:1784948870.773019:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948870.773022:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000048fd39a3. 00000400:00000010:0.0:1784948870.773048:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f7a7941e. 00000800:00000001:0.0:1784948870.773060:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948870.773063:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948871.412010:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948871.412035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948871.412045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948871.412047:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948871.412118:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948871.412123:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948871.412127:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948871.604072:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948871.604110:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948871.604113:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948871.604121:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948871.604126:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948871.604131:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948871.604135:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948872.436147:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948872.436155:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948872.436162:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948872.436163:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948872.436241:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948872.436249:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948872.436253:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948872.627996:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948872.628008:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948872.628042:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948847/real 1784948847] req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948872 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784948872.628067:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948872.628070:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948872.628075:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ebd65351. 00000400:00000200:0.0:1784948872.628086:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000100:00000040:1.0:1784948872.628087:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1784948872.628090:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000100:00000001:1.0:1784948872.628098:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948872.628099:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948872.628103:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948872 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784948872.628111:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb5afc66. 00000100:00000200:0.0:1784948872.628112:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948872 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784948872.628120:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948872.628123:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948872.628126:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948872.628128:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784948872.628130:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948872.628133:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948872.628134:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1784948872.628137:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948872 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948872.628144:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.628146:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784948872.628157:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000adf08f9b. 00000100:00000001:0.0:1784948872.628159:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948872.628163:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.628165:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948872.628168:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.628170:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948872.628174:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948872.628178:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784948872.628181:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948872 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948872.628209:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948872.628210:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784948872.628215:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.628216:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.628222:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948872.628223:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.628226:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:1.0:1784948872.628228:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb5afc66] to pc [ptlrpcd_00_03+3] req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948872.628230:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948872.628237:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1784948872.628303:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948872.628307:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948872.628314:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1784948872.628315:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:1.0:1784948872.628320:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1784948872.628326:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1784948872.628326:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948872.628335:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784948872.628344:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11119 00000100:00000040:0.0:1784948872.628348:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948872 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948872.628364:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:1.0:1784948872.628377:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11118 00000100:00100000:0.0:1784948872.628385:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289656960:192.168.202.137@tcp:8: 00000100:00000001:3.0:1784948872.628403:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948872.628410:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:3.0:1784948872.628413:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948872.628419:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:3.0:1784948872.628421:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948872.628429:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000040:3.0:1784948872.628437:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948872.628439:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948872.628448:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948872.628458:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:2.0:1784948872.628470:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948872.628486:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948872.628486:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1784948872.628499:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:3.0:1784948872.628500:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948872.628501:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948872.628510:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948872.628525:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:3.0:1784948872.628525:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1784948872.628531:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784948872.628532:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948872.628537:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784948872.628539:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948872.628542:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000040:1.0:1784948872.628550:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00100000:3.0:1784948872.628550:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb5afc66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289658240:0@lo:400: 00000100:00000001:1.0:1784948872.628559:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1784948872.628562:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000040:1.0:1784948872.628586:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948872.628586:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784948872.628593:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:2.0:1784948872.628593:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000041892d66. 02000000:00000001:3.0:1784948872.628597:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948872.628608:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948872.628612:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:3.0:1784948872.628615:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:2.0:1784948872.628616:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00080000:1.0:1784948872.628628:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00080000:1.0:1784948872.628634:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 02000000:00000010:3.0:1784948872.628636:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000898cc47a. 00000100:00000001:1.0:1784948872.628640:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948872.628640:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000da72cb2d. 00000100:00000001:0.0:1784948872.628645:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948872.628646:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948872.628671:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1784948872.628672:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1784948872.628673:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4db7024. 00000400:00000200:2.0:1784948872.628687:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000040:0.0:1784948872.628693:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb4f6eba x1871651289656960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948872 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948872.628694:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948872.628706:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948872.628708:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000001:1.0:1784948872.628716:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:3.0:1784948872.628725:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289658240, portal 25 00000100:00000001:1.0:1784948872.628726:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948872.628728:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:0.0:1784948872.628733:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784948872.628747:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1784948872.628749:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000010:1.0:1784948872.628751:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000096f3921c. 00000100:00000001:3.0:1784948872.628754:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549154432 : -115544160397184 : ffff96e9c75d1880) 02000000:00000010:0.0:1784948872.628756:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000af0b2fc8. 00000400:00000200:2.0:1784948872.628762:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:1.0:1784948872.628763:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=4 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784948872.628788:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784948872.628789:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948872.628794:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1784948872.628796:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:0.0:1784948872.628797:0:337118:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:1.0:1784948872.628798:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948872.628812:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:1.0:1784948872.628812:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948872.628819:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1784948872.628820:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784948872.628821:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784948872.628827:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1784948872.628829:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948872.628830:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000009e9a21e3. 00000100:00000040:3.0:1784948872.628835:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948872.628847:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 02000000:00000001:0.0:1784948872.628849:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:1.0:1784948872.628851:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000009265644e. 00000400:00000200:2.0:1784948872.628854:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:0.0:1784948872.628855:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000200:3.0:1784948872.628857:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289658240, offset 0 00000100:00000001:1.0:1784948872.628858:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948872.628861:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb4f6eba. 00000100:00100000:1.0:1784948872.628873:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948872.628875:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000400:00000200:2.0:1784948872.628878:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3341:3341:241:4294967295] 1.2.3.4@tcp LPNI seq info [100:100:8:4294967295] 00000100:00000001:0.0:1784948872.628881:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948872.628886:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948872.628894:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.628908:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948872.628909:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948872.628911:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948872.628913:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.628916:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948872.628920:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096f3921c] to pc [ptlrpcd_rcv+-1] req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948872.628932:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948872.628935:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948872.628939:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948872.628945:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a612e3ff. 00000100:00000001:0.0:1784948872.628946:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948872.628951:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948872.628956:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948872.628967:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948872.628970:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948872.628976:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784948872.628977:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000020:00000040:1.0:1784948872.628983:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 02000000:00000001:0.0:1784948872.628989:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948872.628991:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948872.628995:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948872.628996:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948872.628998:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948872.629003:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000096f3921c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289658368:1.2.3.4@tcp:8: 02000000:00000010:1.0:1784948872.629006:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:0.0:1784948872.629022:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948872.629026:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.629028:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.629030:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.629038:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948872.629039:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.629055:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784948872.629057:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948872.629065:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948872.629065:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:0.0:1784948872.629081:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948872.629090:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948872.629102:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948872.629105:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948872.629106:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948872.629115:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948872.629119:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000400:00000200:3.0:1784948872.629129:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:1.0:1784948872.629140:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a612e3ff] to pc [ptlrpcd_00_00+0] req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1784948872.629143:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005a1c05a3. 00000400:00000200:3.0:1784948872.629155:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437380 00000800:00000200:2.0:1784948872.629171:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:3.0:1784948872.629187:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7c25 [2] + 0 00000400:00000010:3.0:1784948872.629193:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000084252b45. 00000100:00000001:1.0:1784948872.629198:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1784948872.629200:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:3.0:1784948872.629208:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d51a2e44 00000100:00000040:1.0:1784948872.629213:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11149) 00000400:00000200:2.0:1784948872.629216:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:0.0:1784948872.629221:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784948872.629223:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948872.629236:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948872.629238:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11149) 00000400:00000200:2.0:1784948872.629242:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 00000100:00000001:0.0:1784948872.629263:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1784948872.629269:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948872.629289:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d51a2e44 00000100:00000040:0.0:1784948872.629290:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948872.629300:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d51a2e44. 00000100:00000001:1.0:1784948872.629304:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948872.629315:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.629316:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784948872.629317:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:3.0:1784948872.629326:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784948872.629327:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948872.629337:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000032a4c2ab x1871651289658240 msgsize 224 02000000:00000001:0.0:1784948872.629338:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948872.629344:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948872.629353:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948872.629355:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784948872.629361:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00100000:0.0:1784948872.629363:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289658496:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948872.629374:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948872.629377:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948872.629379:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948872.629381:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948872.629386:0:337122:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:0.0:1784948872.629389:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 02000000:00000001:0.0:1784948872.629393:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948872.629400:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ebd65351. 00000400:00000200:3.0:1784948872.629401:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000400:00000010:3.0:1784948872.629407:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:3.0:1784948872.629412:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:0.0:1784948872.629419:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 00000100:00000200:3.0:1784948872.629423:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948872.629430:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289658496, portal 10 00000100:00000001:0.0:1784948872.629454:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948872.629457:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549158016 : -115544160393600 : ffff96e9c75d2680) 00000100:00000001:2.0:1784948872.629461:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948872.629462:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948872.629474:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948872.629476:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289658240 00000100:00000040:3.0:1784948872.629477:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948872.629494:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948872.629504:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948872.629520:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.629538:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948872.629539:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.629543:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.629559:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1784948872.629561:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00100000:2.0:1784948872.629562:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289658240 00000100:00000001:3.0:1784948872.629575:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948872.629580:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000020:00000001:2.0:1784948872.629580:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948872.629586:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305812011 00000020:00000040:2.0:1784948872.629597:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000200:0.0:1784948872.629600:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289658496, offset 0 00000100:00000010:3.0:1784948872.629603:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000020:00000001:2.0:1784948872.629606:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:0.0:1784948872.629618:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000040:2.0:1784948872.629618:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948872.629624:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1784948872.629633:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784948872.629640:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784948872.629649:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948872.629651:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948872.629652:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305812011 00000400:00000200:0.0:1784948872.629665:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:3.0:1784948872.629669:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000006204bf6f. 00000020:00000010:2.0:1784948872.629672:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000400:00000200:0.0:1784948872.629677:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948872.629681:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948872.629690:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948872.629692:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948872.629700:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305812011 00000400:00000200:0.0:1784948872.629710:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:2.0:1784948872.629713:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00000010:3.0:1784948872.629714:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000ead0e753. 00000400:00000200:0.0:1784948872.629722:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3342:3342:240:4294967295] 192.168.202.137@tcp LPNI seq info [2158:2158:8:4294967295] 00000100:00000001:3.0:1784948872.629732:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948872.629742:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305812011 00000020:00000010:2.0:1784948872.629744:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e5260462. 00000100:00000010:3.0:1784948872.629754:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000400:00000200:0.0:1784948872.629780:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:2.0:1784948872.629780:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000091201221. 00000100:00000001:3.0:1784948872.629781:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948872.629793:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948872.629799:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948872.629801:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006782f249. 00000800:00000200:0.0:1784948872.629803:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948872.629803:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.629814:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784948872.629816:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000100:00000001:3.0:1784948872.629822:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948872.629826:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:0.0:1784948872.629830:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948872.629836:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948872.629837:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948872.629839:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:0.0:1784948872.629847:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784948872.629847:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948872.629852:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948872.629854:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948872.629855:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948872.629865:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948872.629872:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.629882:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948872.629888:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948872.629890:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2071 usecs (3 jiffies) 00000100:00000001:0.0:1784948872.629896:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305811972 00000100:00000001:0.0:1784948872.629900:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305812011 00000100:00000001:0.0:1784948872.629902:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305811972 00000100:00000001:0.0:1784948872.629933:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305812011 00000100:00000001:0.0:1784948872.629937:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305811973 00000100:00000001:0.0:1784948872.629940:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305812011 00000100:00000001:0.0:1784948872.629944:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305811973 00000100:00000001:0.0:1784948872.629947:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305812011 00000100:00000001:0.0:1784948872.629948:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948872.629951:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.629952:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.629970:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948872.629974:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.629976:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948872.629978:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.629990:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948872.629993:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1784948872.629995:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 723 00000100:00000001:0.0:1784948872.630000:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948872.630002:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948872.630003:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948872.630023:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782149712 : -115538927401904 : ffff96eaff464450) 00000100:00000001:0.0:1784948872.630025:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.630029:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948872.630032:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000032a4c2ab x1871651289658240/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:533/0 lens 224/0 e 0 to 0 dl 1784948883 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948872.630042:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948872.630047:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000002e6ac4ed. 00000100:00000001:3.0:1784948872.630049:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948872.630051:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000002e6ac4ed. 00000100:00100000:3.0:1784948872.630056:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000032a4c2ab pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289658240:12345-0@lo:400: 00000100:00000200:3.0:1784948872.630060:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289658240 00000020:00000001:3.0:1784948872.630065:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948872.630071:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948872.630075:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948872.630077:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948872.630080:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784948872.630085:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948872.630086:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948872.630089:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948872.630091:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948872.630093:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948872.630097:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948872.630100:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948872.630103:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948872.630108:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008d04782c. 02000000:00000001:3.0:1784948872.630111:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.630113:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948872.630116:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948872.630119:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289658240 00000800:00000200:1.0:1784948872.630122:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1784948872.630123:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000010:1.0:1784948872.630130:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00010000:00000200:3.0:1784948872.630131:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000032a4c2ab x1871651289658240/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:533/0 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948872.630137:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948872.630138:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:3.0:1784948872.630149:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:1.0:1784948872.630150:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948872.630154:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784948872.630154:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948872.630156:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:1.0:1784948872.630163:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000800:00000001:0.0:1784948872.630164:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:3.0:1784948872.630165:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=44 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948872.630170:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:0.0:1784948872.630172:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948872.630172:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1784948872.630178:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948872.630179:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:0.0:1784948872.630186:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948872.630186:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948872.630192:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948872.630197:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948872.630203:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437480 00000100:00000001:1.0:1784948872.630203:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948872.630204:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948872.630214:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948872.630214:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:0.0:1784948872.630219:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 157616 00000400:00000010:3.0:1784948872.630221:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000800:00000001:0.0:1784948872.630233:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948872.630234:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948872.630235:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289658240, offset 224 00000100:00000001:1.0:1784948872.630241:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784948872.630244:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948872.630248:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948872.630250:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948872.630276:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948872.630280:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948872.630287:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948872.630292:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:3.0:1784948872.630296:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437380 00000100:00000010:0.0:1784948872.630302:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d0fcbe33. 00000400:00000200:3.0:1784948872.630310:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x862d [1] + 224 00000100:00000040:0.0:1784948872.630311:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000d0fcbe33 x1871651289658496 msgsize 224 00000100:00100000:0.0:1784948872.630324:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:3.0:1784948872.630326:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948872.630332:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948872.630338:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948872.630342:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948872.630345:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784948872.630359:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948872.630360:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948872.630368:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948872.630400:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948872.630402:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1784948872.630406:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289658496 02000000:00000001:1.0:1784948872.630411:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:3.0:1784948872.630414:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000100:00000001:1.0:1784948872.630415:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:3.0:1784948872.630421:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:1.0:1784948872.630422:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.630447:0:342755:0:(events.c:386:reply_out_callback()) Process entered 02000000:00000001:1.0:1784948872.630449:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948872.630452:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00100000:1.0:1784948872.630454:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289658496 00000020:00000001:1.0:1784948872.630459:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948872.630462:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:3.0:1784948872.630462:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948872.630468:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:3.0:1784948872.630473:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000040:1.0:1784948872.630476:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784948872.630477:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1784948872.630479:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948872.630487:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948872.630487:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784948872.630488:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948872.630516:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:1.0:1784948872.630519:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000001:2.0:1784948872.630519:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1784948872.630528:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000032a4c2ab x1871651289658240/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:533/0 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784948872.630532:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784948872.630536:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:2.0:1784948872.630540:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000020:00000010:1.0:1784948872.630546:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f7efedb6. 00000100:00000001:1.0:1784948872.630560:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784948872.630565:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000020:00000010:1.0:1784948872.630568:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f7efedb6. 00000100:00100000:3.0:1784948872.630578:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000032a4c2ab pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289658240:12345-0@lo:400: Request processed in 523us (1227us total) trans 0 rc 0/0 00000400:00000010:2.0:1784948872.630580:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000020:00000010:1.0:1784948872.630588:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bf7eadde. 00000100:00000001:2.0:1784948872.630599:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:1.0:1784948872.630605:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ae8262ed. 00000100:00100000:3.0:1784948872.630616:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 723 00000100:00000200:2.0:1784948872.630617:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948872.630622:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:3.0:1784948872.630638:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000040:1.0:1784948872.630649:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948872.630655:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948872.630669:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948872.630670:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:1.0:1784948872.630678:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:2.0:1784948872.630680:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948872.630685:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000091201221. 00000100:00000001:1.0:1784948872.630687:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948872.630700:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006782f249. 00000100:00000001:1.0:1784948872.630705:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948872.630733:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000010:3.0:1784948872.630734:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e5260462. 00000100:00000001:1.0:1784948872.630739:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948872.630752:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784948872.630756:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1784948872.630762:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948872.630763:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948872.630768:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784948872.630768:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008d04782c. 00000100:00000001:2.0:1784948872.630789:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948872.630796:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:2.0:1784948872.630799:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948872.630801:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948872.630804:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948872.630811:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948872.630812:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:3.0:1784948872.630814:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:1.0:1784948872.630821:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 699 00000100:00000001:2.0:1784948872.630821:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948872.630823:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000084252b45. 00000400:00000010:3.0:1784948872.630841:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000042334b2c. 00000100:00000040:1.0:1784948872.630847:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:2.0:1784948872.630847:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948872.630867:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558292224 : -115544151259392 : ffff96e9c7e88700) 00000100:00001000:2.0:1784948872.630872:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=44 v=5 (5 5 5 5) 00000100:00000040:1.0:1784948872.630876:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d0fcbe33 x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:533/0 lens 224/0 e 0 to 0 dl 1784948883 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:2.0:1784948872.630888:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=44 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948872.630910:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948872.630913:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f7efedb6. 00000100:00000001:2.0:1784948872.630955:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948872.630958:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948872.630959:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784948872.630967:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:1.0:1784948872.630969:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f7efedb6. 00010000:00000001:2.0:1784948872.630978:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948872.630989:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d0fcbe33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289658496:12345-192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948872.630989:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948872.631027:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289658496 00000020:00000001:1.0:1784948872.631032:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000040:2.0:1784948872.631034:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948872.631036:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948872.631042:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948872.631056:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948872.631061:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948872.631072:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948872.631079:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948872.631081:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948872.631084:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948872.631096:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:2.0:1784948872.631097:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1784948872.631111:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948872.631111:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948872.631135:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:2.0:1784948872.631142:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948872.631145:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948872.631154:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948872.631164:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c45ab905. 02000000:00000001:1.0:1784948872.631176:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948872.631183:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb5afc66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289658240:0@lo:400: 00000100:00000001:1.0:1784948872.631186:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948872.631202:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948872.631207:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1784948872.631208:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289658496 00010000:00000001:1.0:1784948872.631229:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784948872.631229:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948872.631252:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb5afc66 x1871651289658240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784948872.631270:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d0fcbe33 x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:533/0 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948872.631295:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948872.631304:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948872.631319:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000898cc47a. 00010000:00000001:1.0:1784948872.631325:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:2.0:1784948872.631327:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00010000:00000001:1.0:1784948872.631330:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948872.631332:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948872.631348:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00001000:1.0:1784948872.631349:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=39 v=5 (1 1 1 1) 00000020:00000001:2.0:1784948872.631362:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:1.0:1784948872.631365:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:2.0:1784948872.631368:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000adf08f9b. 00000100:00000040:1.0:1784948872.631375:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948872.631380:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948872.631380:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948872.631386:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1784948872.631388:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000010:2.0:1784948872.631390:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb5afc66. 02000000:00000001:1.0:1784948872.631393:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948872.631399:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948872.631401:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948872.631413:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948872.631424:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948872.631426:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948872.631453:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948872.631457:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948872.631458:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000100:00000200:1.0:1784948872.631473:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289658496, offset 224 00000100:00000001:2.0:1784948872.631473:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948872.631483:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948872.631490:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948872.631500:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948872.631508:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3343:3343:240:4294967295] 192.168.202.137@tcp LPNI seq info [2159:2159:8:4294967295] 00000400:00000200:1.0:1784948872.631519:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948872.631523:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948872.631526:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000031deb17e. 00000800:00000200:1.0:1784948872.631531:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948872.631536:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948872.631538:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000031deb17e type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948872.631551:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948872.631554:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948872.631556:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948872.631559:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948872.631560:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948872.631563:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d0fcbe33 x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:533/0 lens 224/224 e 0 to 0 dl 1784948883 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948872.631572:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d0fcbe33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289658496:12345-192.168.202.137@tcp:400: Request processed in 662us (1260us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948872.631578:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 699 00000100:00000040:1.0:1784948872.631580:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948872.631582:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948872.631583:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948872.631586:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ae8262ed. 00000020:00000010:1.0:1784948872.631589:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948872.631591:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bf7eadde. 00000020:00000040:1.0:1784948872.631595:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948872.631597:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948872.631733:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948872.631740:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000031deb17e. 00000400:00000200:0.0:1784948872.631748:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948872.631750:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948872.631758:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948872.631766:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000800:00000001:1.0:1784948872.631766:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948872.631788:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000800:00000001:1.0:1784948872.631789:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948872.631795:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784948872.631795:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.631799:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948872.631805:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948872.631815:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437480 00000400:00000200:1.0:1784948872.631820:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8635 [1] + 224 00000800:00000001:1.0:1784948872.631825:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948872.631832:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948872.631834:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948872.631837:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948872.631841:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948872.631844:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948872.631851:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948872.631872:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948872.631876:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948872.631878:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.631903:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948872.631925:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948872.631930:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948872.631931:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948872.631933:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ebd65351. 00000400:00000200:0.0:1784948872.631937:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000400:00000010:0.0:1784948872.631938:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000100:00000001:0.0:1784948872.631940:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948872.631945:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948872.631954:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948872.631960:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948872.631962:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.631964:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948872.631966:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948872.631968:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948872.631970:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948872.631972:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948872.631978:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=39 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948872.631981:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=39 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948872.631983:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948872.631986:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948872.631988:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948872.631991:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948872.631998:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.632002:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948872.632003:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948872.632004:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948872.632005:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948872.632025:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948872.632032:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948872.632034:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948872.632035:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948872.632038:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948872.632043:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289658496:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948872.632047:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.632048:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948872.632050:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a612e3ff x1871651289658496/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948888 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948872.632055:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948872.632057:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948872.632059:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 02000000:00000001:0.0:1784948872.632062:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948872.632064:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948872.632065:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948872.632067:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948872.632069:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 02000000:00000001:0.0:1784948872.632072:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948872.632073:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948872.632074:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a612e3ff. 00000100:00000001:0.0:1784948872.632078:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948872.632079:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.632081:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948872.632084:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948872.632085:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948872.632087:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948872.632089:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948872.672073:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305812011, at 4305812012 00000100:00000040:0.0:1784948872.672075:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305812011, at 4305812012 00000100:00000040:3.0:1784948872.672084:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305812011, at 4305812012 00000100:00000010:0.0:1784948872.672096:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00000100:00000010:3.0:1784948872.672099:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000010:1.0:1784948872.672106:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000006204bf6f. 00000100:00000040:2.0:1784948872.672127:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305812011, at 4305812012 00000100:00000010:2.0:1784948872.672149:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000ead0e753. 00010000:00000001:2.0:1784948873.460128:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948873.460143:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948873.460155:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948873.460157:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948873.460296:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948873.460304:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948873.460309:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948873.652091:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948873.652104:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948873.652124:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948872/real 0] req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948873 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948873.652144:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.652148:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948873.652151:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948873.652156:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948873 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948873.652166:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.652178:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948873.652180:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948873.652184:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948873.652188:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948873.652192:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948873.652198:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948873 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948873.652211:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784948873.652216:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.652218:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948873.652221:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.652226:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784948873.652231:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948873.652237:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784948873.652276:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784948873.652282:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784948873.652289:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948873 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948873.652300:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000096f3921c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289658368:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784948873.652308:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.652313:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948873.652318:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000096f3921c x1871651289658368/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948873 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948873.652318:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948873.652335:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948873.652340:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784948873.652345:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000010:3.0:1784948873.652346:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000095d61ef9. 00000020:00000040:0.0:1784948873.652351:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000040:3.0:1784948873.652355:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948873.652356:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948873.652365:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000009265644e. 00000100:00000001:3.0:1784948873.652365:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948873.652371:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948873.652376:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948873.652379:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948873.652380:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948873.652385:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000096f3921c. 00000100:00000001:0.0:1784948873.652390:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000010:3.0:1784948873.652392:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:0.0:1784948873.652393:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.652400:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.652405:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948873.652406:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948873.652407:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948873.652416:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948873.652419:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948873.652422:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948873.652437:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948873.652440:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948873.652474:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095d61ef9] to pc [ptlrpcd_00_01+1] req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948873.652518:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948873.652521:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948873.652525:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948873.652529:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784948873.652531:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784948873.652536:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784948873.652541:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11119 00000400:00000200:3.0:1784948873.652555:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:3.0:1784948873.652561:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11144 00000400:00000200:3.0:1784948873.652567:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:0.0:1784948873.652568:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784948873.652573:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1784948873.652576:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:1.0:1784948873.652579:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948873.652581:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948873.652583:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948873.652592:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948873.652595:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784948873.652598:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948873.652601:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948873.652604:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784948873.652606:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948873.652612:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784948873.652613:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948873.652616:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1784948873.652628:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:3.0:1784948873.652634:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784948873.652637:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784948873.652641:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:3.0:1784948873.652648:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:3.0:1784948873.652656:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:3.0:1784948873.652667:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948873.652674:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948873.652681:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948873.652689:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948873.652690:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784948873.652694:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948873.652697:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:0.0:1784948873.652702:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1784948873.652705:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 02000000:00000001:0.0:1784948873.652708:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1784948873.652714:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 02000000:00000001:0.0:1784948873.652716:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948873.652725:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948873.652727:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000095d61ef9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289658624:0@lo:400: 00000100:00000001:1.0:1784948873.652731:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948873.652732:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1784948873.652737:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948873.652745:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948873.652749:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948873.652751:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948873.652754:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948873.652756:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:0.0:1784948873.652760:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948873.652762:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948873.652769:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 00000100:00000010:3.0:1784948873.652771:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000e7e08a5. 02000000:00000001:0.0:1784948873.652777:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948873.652779:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784948873.652786:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784948873.652788:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948873.652789:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ebd65351. 00000100:00000001:3.0:1784948873.652792:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948873.652797:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948873.652800:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948873.652803:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 02000000:00000010:3.0:1784948873.652807:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008d04782c. 00000100:00000001:3.0:1784948873.652815:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948873.652816:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289658624, portal 25 00000100:00000001:0.0:1784948873.652822:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1784948873.652826:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948873.652827:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482524288 : -115544227027328 : ffff96e9c3646680) 00000100:00000040:0.0:1784948873.652842:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948873.652850:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948873.652855:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948873.652864:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948873.652866:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000e7e08a5] to pc [ptlrpcd_rcv+-1] req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948873.652873:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948873.652881:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000100:00000200:0.0:1784948873.652888:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289658624, offset 0 00000400:00000200:0.0:1784948873.652899:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948873.652940:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948873.652954:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948873.652958:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784948873.652963:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005dccb977. 00000400:00000200:0.0:1784948873.652967:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000040:3.0:1784948873.652969:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948873.652975:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948873.652977:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948873.652979:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948873.652984:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000400:00000200:0.0:1784948873.652987:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437500 00000100:00000001:3.0:1784948873.652990:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948873.652999:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784948873.653029:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7c45 [2] + 0 00000400:00000010:0.0:1784948873.653049:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008b17e867. 00000100:00000001:3.0:1784948873.653054:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948873.653058:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948873.653060:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000004f7481a9 00000100:00000040:3.0:1784948873.653066:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005dccb977] to pc [ptlrpcd_00_02+2] req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948873.653077:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948873.653084:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948873.653093:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f7481a9 00000400:00000010:0.0:1784948873.653097:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000004f7481a9. 00000100:00000001:3.0:1784948873.653100:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948873.653105:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784948873.653109:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11150) 00000100:00000200:0.0:1784948873.653110:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948873.653120:0:337119:0:(events.c:337:request_in_callback()) incoming req@000000003bfcf238 x1871651289658624 msgsize 224 00000100:00100000:0.0:1784948873.653126:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948873.653129:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948873.653146:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948873.653159:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000400:00000010:0.0:1784948873.653162:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:2.0:1784948873.653165:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948873.653170:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948873.653172:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948873.653174:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:0.0:1784948873.653181:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948873.653182:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948873.653183:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948873.653190:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948873.653197:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948873.653216:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948873.653220:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948873.653225:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948873.653229:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948873.653232:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948873.653237:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948873.653244:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948873.653250:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784948873.653254:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289658880:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948873.653259:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.653271:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948873.653276:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948873.653279:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948873.653279:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784948873.653286:0:337119:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784948873.653286:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948873.653289:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:2.0:1784948873.653297:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1784948873.653298:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289658624 00000100:00000001:0.0:1784948873.653301:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948873.653309:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000010:2.0:1784948873.653312:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000053ab16eb. 00000100:00000001:0.0:1784948873.653315:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305813035 00000100:00000001:3.0:1784948873.653316:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948873.653322:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948873.653325:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.653328:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:0.0:1784948873.653330:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 02000000:00000001:3.0:1784948873.653330:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.653336:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948873.653340:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000086a2279a. 00000100:00100000:3.0:1784948873.653343:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289658624 02000000:00000001:1.0:1784948873.653344:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784948873.653347:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1784948873.653354:0:342756:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784948873.653355:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784948873.653355:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:0.0:1784948873.653360:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305813035 00000020:00000040:3.0:1784948873.653362:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 02000000:00000001:1.0:1784948873.653368:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948873.653370:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:0.0:1784948873.653373:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000200:2.0:1784948873.653373:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289658880, portal 10 00000020:00000040:3.0:1784948873.653393:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00100000:1.0:1784948873.653396:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000e7e08a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289658752:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784948873.653402:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948873.653403:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1784948873.653405:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948873.653411:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482519808 : -115544227031808 : ffff96e9c3645500) 00000100:00000001:1.0:1784948873.653415:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948873.653417:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305813035 00000020:00000001:3.0:1784948873.653418:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:1.0:1784948873.653424:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:0.0:1784948873.653429:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 02000000:00000001:1.0:1784948873.653431:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948873.653434:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784948873.653438:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948873.653441:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:1.0:1784948873.653442:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:3.0:1784948873.653446:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784948873.653451:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305813035 02000000:00000010:1.0:1784948873.653452:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000002afa749d. 00000100:00000001:3.0:1784948873.653459:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948873.653465:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948873.653467:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000020:00000010:3.0:1784948873.653471:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784948873.653481:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000010:1.0:1784948873.653482:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d6b8e52f. 00000020:00000010:3.0:1784948873.653490:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000898cc47a. 00000100:00000001:2.0:1784948873.653492:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948873.653495:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948873.653498:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000100:00000001:0.0:1784948873.653503:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948873.653506:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000eefa0203. 00000100:00000040:2.0:1784948873.653507:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:1.0:1784948873.653512:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289658752, portal 4 00000100:00000001:0.0:1784948873.653518:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948873.653520:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000100:00000001:1.0:1784948873.653521:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000010:3.0:1784948873.653523:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002470393a. 00000100:00000001:0.0:1784948873.653525:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.653527:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482523392 : -115544227028224 : ffff96e9c3646300) 00000100:00000200:2.0:1784948873.653533:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289658880, offset 0 00000100:00000040:3.0:1784948873.653551:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000040:1.0:1784948873.653552:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948873.653554:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948873.653562:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948873.653569:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948873.653576:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948873.653578:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784948873.653581:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948873.653594:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948873.653596:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948873.653597:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000010:1.0:1784948873.653602:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000100:00000200:1.0:1784948873.653610:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289658752, offset 0 00000400:00000200:2.0:1784948873.653612:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948873.653623:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948873.653630:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948873.653631:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948873.653648:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784948873.653657:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3344:3344:240:4294967295] 192.168.202.137@tcp LPNI seq info [2160:2160:8:4294967295] 00000100:00000001:0.0:1784948873.653664:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948873.653666:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948873.653671:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948873.653672:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948873.653689:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 724 00000400:00000200:2.0:1784948873.653689:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:0.0:1784948873.653703:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948873.653704:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948873.653712:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948873.653713:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742039632 : -115538967511984 : ffff96eafce23c50) 00000400:00000200:1.0:1784948873.653715:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784948873.653726:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948873.653733:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003bfcf238 x1871651289658624/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:534/0 lens 224/0 e 0 to 0 dl 1784948884 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948873.653736:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000010:2.0:1784948873.653740:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000353a5c12. 00000400:00000200:1.0:1784948873.653748:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948873.653764:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000800:00000200:2.0:1784948873.653766:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948873.653767:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784948873.653778:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:0.0:1784948873.653779:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:0.0:1784948873.653788:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948873.653795:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3345:3345:239:4294967295] 192.168.202.137@tcp LPNI seq info [2161:2161:7:4294967295] 00000800:00000200:2.0:1784948873.653797:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000010:0.0:1784948873.653799:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000800:00000200:2.0:1784948873.653812:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000353a5c12 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:0.0:1784948873.653820:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003bfcf238 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289658624:12345-0@lo:400: 00000100:00000200:0.0:1784948873.653834:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289658624 00000400:00000200:1.0:1784948873.653840:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:0.0:1784948873.653844:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784948873.653847:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948873.653853:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000200:1.0:1784948873.653858:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784948873.653859:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2073 usecs (3 jiffies) 00000020:00000001:0.0:1784948873.653861:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.653873:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305812996 00000800:00000010:1.0:1784948873.653874:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000020:00000001:0.0:1784948873.653875:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948873.653882:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784948873.653885:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305813035 00000020:00000001:0.0:1784948873.653889:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948873.653895:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948873.653896:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784948873.653897:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305812996 00000020:00000001:0.0:1784948873.653902:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784948873.653906:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305813035 00000100:00000001:2.0:1784948873.653909:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305812996 00000100:00000001:2.0:1784948873.653912:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305813035 00000100:00000001:2.0:1784948873.653916:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305812996 00000020:00000001:0.0:1784948873.653944:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948873.653949:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948873.653956:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000020:00000001:0.0:1784948873.653959:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000200:1.0:1784948873.653965:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1784948873.653968:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948873.653971:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305813035 02000000:00000001:0.0:1784948873.653977:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948873.653978:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948873.653985:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948873.653986:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a868d044. 00000100:00000001:2.0:1784948873.653990:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.653993:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948873.653994:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.653997:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.654046:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948873.654047:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:2.0:1784948873.654052:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784948873.654053:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.654056:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305813036 00000100:00000001:2.0:1784948873.654058:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00010000:00000040:0.0:1784948873.654062:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289658624 00000100:00000001:2.0:1784948873.654069:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784948873.654072:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784948873.654072:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305813035 00000100:00000001:2.0:1784948873.654075:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948873.654082:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:0.0:1784948873.654084:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003bfcf238 x1871651289658624/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:534/0 lens 224/224 e 0 to 0 dl 1784948884 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948873.654085:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305813036 00000100:00000001:2.0:1784948873.654089:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.654096:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305813035 00000100:00000001:1.0:1784948873.654110:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305813036 00010000:00000001:0.0:1784948873.654112:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948873.654118:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.654121:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305813035 00000100:00001000:0.0:1784948873.654125:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=45 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948873.654132:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305813036 00000100:00000001:0.0:1784948873.654134:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948873.654141:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948873.654143:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305813035 00000800:00000200:3.0:1784948873.654144:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948873.654145:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:2.0:1784948873.654157:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784948873.654157:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000353a5c12. 02000000:00000001:0.0:1784948873.654158:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784948873.654160:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948873.654165:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948873.654169:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948873.654171:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948873.654172:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:2.0:1784948873.654174:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948873.654177:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948873.654185:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:2.0:1784948873.654186:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948873.654188:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948873.654190:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948873.654195:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948873.654199:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000100:00000001:1.0:1784948873.654201:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948873.654208:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:3.0:1784948873.654209:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000200:2.0:1784948873.654218:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:3.0:1784948873.654219:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000100:00000001:1.0:1784948873.654220:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948873.654227:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948873.654231:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948873.654233:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948873.654234:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289658624, offset 224 00000100:00000001:1.0:1784948873.654241:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948873.654245:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948873.654251:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437600 00000400:00000200:0.0:1784948873.654260:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948873.654275:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948873.654281:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948873.654289:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784948873.654290:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 157840 00000100:00000001:3.0:1784948873.654311:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948873.654314:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948873.654317:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948873.654321:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437500 00000800:00000001:3.0:1784948873.654331:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948873.654339:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8649 [1] + 224 00000800:00000001:3.0:1784948873.654345:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948873.654351:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948873.654352:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948873.654360:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784948873.654362:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948873.654363:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000010:1.0:1784948873.654368:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000100:00000001:0.0:1784948873.654378:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1784948873.654379:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948873.654381:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:0.0:1784948873.654390:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948873.654397:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948873.654409:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948873.654410:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6437580 00000400:00000200:1.0:1784948873.654413:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000200:2.0:1784948873.654421:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1784948873.654423:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000100:00000040:0.0:1784948873.654430:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948873.654435:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784948873.654438:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:1.0:1784948873.654449:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000100:3.0:1784948873.654453:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289658752 offset 0 length 520: 4 00000100:00000001:2.0:1784948873.654454:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948873.654460:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:3.0:1784948873.654471:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000da3ac487 is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000010:2.0:1784948873.654476:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a1f38785. 00000400:00000200:3.0:1784948873.654486:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000040:2.0:1784948873.654488:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000a1f38785 x1871651289658880 msgsize 224 00000100:00000001:0.0:1784948873.654497:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948873.654505:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:3.0:1784948873.654510:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784948873.654511:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:0.0:1784948873.654514:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000100:00000040:1.0:1784948873.654521:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948898 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948873.654522:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000800:00000001:3.0:1784948873.654530:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.654533:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948873.654534:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948873.654543:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948873.654544:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948873.654553:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948873.654554:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948873.654555:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948873.654556:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948873.654563:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948873.654569:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1784948873.654570:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784948873.654576:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948873.654580:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948873.654584:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948873.654592:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003bfcf238 x1871651289658624/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:534/0 lens 224/224 e 0 to 0 dl 1784948884 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948873.654598:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948873.654607:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289658880 00000100:00000001:3.0:1784948873.654610:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948873.654612:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003bfcf238 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289658624:12345-0@lo:400: Request processed in 839us (1485us total) trans 0 rc 0/0 02000000:00000001:2.0:1784948873.654614:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948873.654616:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948873.654621:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948873.654628:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784948873.654628:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948873.654636:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1784948873.654637:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 724 02000000:00000001:2.0:1784948873.654638:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948873.654644:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ebd65351. 00000100:00100000:2.0:1784948873.654648:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289658880 00000100:00000040:0.0:1784948873.654650:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:3.0:1784948873.654652:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000020:00000001:2.0:1784948873.654658:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948873.654660:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:3.0:1784948873.654661:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000020:00000040:2.0:1784948873.654662:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784948873.654666:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:2.0:1784948873.654672:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948873.654674:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784948873.654677:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000eefa0203. 00000020:00000040:2.0:1784948873.654679:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:3.0:1784948873.654685:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948873.654687:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000010:0.0:1784948873.654688:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002470393a. 00000020:00000001:2.0:1784948873.654699:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000010:0.0:1784948873.654701:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000898cc47a. 00000100:00000001:2.0:1784948873.654712:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000040:0.0:1784948873.654715:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000010:2.0:1784948873.654718:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000200:3.0:1784948873.654720:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948873.654724:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948873.654727:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1784948873.654731:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a868d044. 00000020:00000010:2.0:1784948873.654738:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 02000000:00000001:0.0:1784948873.654741:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948873.654749:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948873.654753:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a5bcc72e. 00000100:00000001:3.0:1784948873.654756:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:0.0:1784948873.654759:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784948873.654761:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948873.654763:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000da98d9f1. 00000400:00000010:0.0:1784948873.654768:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000010bdacaf. 00000100:00000001:3.0:1784948873.654771:0:337119:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:2.0:1784948873.654775:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 02000000:00000001:3.0:1784948873.654777:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:0.0:1784948873.654778:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000059abd2b8. 00000100:00000001:3.0:1784948873.654785:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948873.654789:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948873.654793:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:3.0:1784948873.654794:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.654801:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948873.654806:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00001000:3.0:1784948873.654806:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=45 v=5 (5 5 5 5) 00000100:00000001:2.0:1784948873.654813:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784948873.654815:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=45 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948873.654823:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948873.654825:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948873.654826:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.654831:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00010000:00000001:3.0:1784948873.654834:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:2.0:1784948873.654835:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948873.654839:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948873.654844:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948873.654849:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948873.654856:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948873.654860:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948873.654860:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948873.654863:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948873.654870:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948873.654881:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784948873.654882:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000095d61ef9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289658624:0@lo:400: 00000100:00000001:2.0:1784948873.654888:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948873.654895:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948873.654936:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948873.654941:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948873.654943:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 700 00000100:00000001:3.0:1784948873.654951:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948873.654952:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948873.654952:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948873.654960:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470028672 : -115544239522944 : ffff96e9c2a5bb80) 00000100:00000040:3.0:1784948873.654962:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000095d61ef9 x1871651289658624/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948873.654963:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948873.654974:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a1f38785 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:534/0 lens 224/0 e 0 to 0 dl 1784948884 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948873.655022:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948873.655028:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948873.655034:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 00000100:00000001:2.0:1784948873.655038:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:3.0:1784948873.655042:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:2.0:1784948873.655045:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000020:00000001:3.0:1784948873.655048:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948873.655054:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948873.655055:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948873.655066:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000010:2.0:1784948873.655067:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 02000000:00000010:3.0:1784948873.655073:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 00000100:00100000:2.0:1784948873.655082:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a1f38785 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289658880:12345-192.168.202.137@tcp:400: 02000000:00000001:3.0:1784948873.655085:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948873.655089:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948873.655093:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000095d61ef9. 00000100:00000200:2.0:1784948873.655095:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289658880 00000100:00000001:3.0:1784948873.655101:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:2.0:1784948873.655102:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784948873.655106:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948873.655109:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784948873.655111:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948873.655115:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948873.655120:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1784948873.655121:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784948873.655127:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948873.655128:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:3.0:1784948873.655137:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948873.655138:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784948873.655142:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948873.655145:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948873.655150:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948873.655151:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948873.655153:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948873.655156:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948873.655160:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948873.655162:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948873.655201:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f429991e. 02000000:00000001:2.0:1784948873.655205:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.655208:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948873.655210:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948873.655213:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289658880 00010000:00000001:2.0:1784948873.655216:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948873.655221:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a1f38785 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:534/0 lens 224/224 e 0 to 0 dl 1784948884 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948873.655231:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948873.655234:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948873.655236:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=40 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948873.655241:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948873.655243:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948873.655246:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948873.655249:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948873.655251:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.655253:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948873.655256:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948873.655260:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000100:00000200:2.0:1784948873.655264:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289658880, offset 224 00000400:00000200:2.0:1784948873.655270:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948873.655278:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948873.655284:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3346:3346:240:4294967295] 192.168.202.137@tcp LPNI seq info [2162:2162:8:4294967295] 00000400:00000200:2.0:1784948873.655295:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948873.655303:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948873.655309:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000635133da. 00000800:00000200:2.0:1784948873.655316:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948873.655326:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784948873.655332:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000635133da type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948873.655354:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948873.655360:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948873.655364:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948873.655367:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948873.655371:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948873.655380:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a1f38785 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:534/0 lens 224/224 e 0 to 0 dl 1784948884 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948873.655400:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a1f38785 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289658880:12345-192.168.202.137@tcp:400: Request processed in 359us (908us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948873.655415:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 700 00000100:00000040:2.0:1784948873.655419:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948873.655423:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948873.655426:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948873.655431:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000da98d9f1. 00000020:00000010:2.0:1784948873.655437:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784948873.655440:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 00000020:00000040:2.0:1784948873.655446:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948873.655451:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948873.655452:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948873.655459:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000635133da. 00000400:00000200:3.0:1784948873.655466:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948873.655475:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948873.655482:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000010:3.0:1784948873.655485:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000800:00000001:0.0:1784948873.655488:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948873.655490:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948873.655495:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948873.655503:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948873.655508:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948873.655512:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948873.655520:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948873.655531:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437600 00000400:00000200:0.0:1784948873.655537:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8651 [1] + 224 00000800:00000001:0.0:1784948873.655543:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948873.655549:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948873.655551:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948873.655556:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948873.655560:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948873.655564:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948873.655575:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948873.655597:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948873.655604:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948873.655607:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948873.655646:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948873.655651:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948873.655656:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948873.655659:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948873.655661:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000086a2279a. 00000400:00000200:2.0:1784948873.655667:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784948873.655669:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00000001:2.0:1784948873.655672:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948873.655678:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948873.655689:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948873.655697:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948873.655699:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948873.655701:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948873.655703:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948873.655705:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948873.655707:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948873.655709:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948873.655715:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=40 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948873.655718:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=40 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948873.655721:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948873.655723:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948873.655725:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948873.655729:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948873.655737:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.655740:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948873.655742:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948873.655744:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948873.655746:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948873.655749:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948873.655757:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948873.655759:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948873.655761:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948873.655764:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948873.655773:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289658880:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948873.655778:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948873.655781:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948873.655783:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005dccb977 x1871651289658880/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948873.655790:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948873.655791:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948873.655793:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000053ab16eb. 02000000:00000001:2.0:1784948873.655797:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948873.655799:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948873.655800:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948873.655802:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948873.655804:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:2.0:1784948873.655808:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948873.655809:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948873.655811:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005dccb977. 00000100:00000001:2.0:1784948873.655814:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948873.655816:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948873.655818:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948873.655822:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948873.655824:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948873.655826:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948873.655828:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948873.696980:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305813036, at 4305813037 00000100:00000010:3.0:1784948873.697019:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000040:1.0:1784948873.697058:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305813036, at 4305813037 00000100:00000040:2.0:1784948873.697069:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305813036, at 4305813037 00000100:00000010:1.0:1784948873.697074:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000040:0.0:1784948873.697075:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305813036, at 4305813037 00000100:00000010:2.0:1784948873.697083:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000010:0.0:1784948873.697090:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00010000:00000001:2.0:1784948874.484105:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948874.484120:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948874.484133:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948874.484137:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948874.484230:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948874.484239:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948874.484245:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000001:1.0:1784948875.087579:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948875.087610:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948875.087621:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948875.087624:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948875.508018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948875.508028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948875.508035:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948875.508036:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948875.508112:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948875.508117:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948875.508121:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948876.532056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948876.532067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948876.532075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948876.532077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948876.532140:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948876.532146:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948876.532150:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948877.556069:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948877.556080:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948877.556091:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948877.556093:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948877.556180:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948877.556185:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948877.556190:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948878.580086:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948878.580098:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948878.580107:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948878.580109:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948878.580185:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948878.580192:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948878.580196:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948878.772100:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948878.772121:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948878.772125:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948878.772132:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948878.772133:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948878.772137:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948878.772146:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.772158:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948878.772161:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948878.772181:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000020:00000040:0.0:1784948878.772193:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948878.772203:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948878.772209:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948878.772219:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948878.772237:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f64a2f6c. 00000100:00000001:0.0:1784948878.772250:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948878.772265:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948878.772285:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948878.772289:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948878.772298:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c39699a8] to pc [ptlrpcd_00_03+3] req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948878.772340:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948878.772345:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948878.772353:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948878.772360:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948878.772365:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948878.772368:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948878.772374:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1784948878.772376:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000100:00000001:3.0:1784948878.772379:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1784948878.772385:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000040:3.0:1784948878.772390:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948878.772394:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784948878.772394:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948878.772401:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948878.772405:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948878.772407:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.772418:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1784948878.772421:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000000d1fbdb0. 00000100:00000001:3.0:1784948878.772422:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948878.772433:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948878.772434:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948878.772439:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948878.772444:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948878.772450:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1784948878.772450:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289659008:0@lo:400: 00000100:00000001:3.0:1784948878.772481:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948878.772492:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784948878.772496:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1784948878.772498:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948878.772504:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948878.772505:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948878.772517:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000898cc47a. 00000100:00000040:0.0:1784948878.772519:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_00_00+0] req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948878.772523:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948878.772539:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a51bb92. 00000400:00000010:3.0:1784948878.772554:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000100:00000200:3.0:1784948878.772565:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659008, portal 25 00000100:00000001:3.0:1784948878.772569:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948878.772572:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558302976 : -115544151248640 : ffff96e9c7e8b100) 00000100:00000040:3.0:1784948878.772581:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948878.772589:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.772595:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948878.772605:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11155) 00000100:00000040:3.0:1784948878.772605:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948878.772613:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00000200:3.0:1784948878.772621:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289659008, offset 0 00000400:00000200:3.0:1784948878.772632:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948878.772637:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948878.772644:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948878.772649:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1784948878.772658:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784948878.772661:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948878.772669:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948878.772673:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948878.772697:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437680 00000100:00000001:1.0:1784948878.772726:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1784948878.772728:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7c5d [2] + 0 00000100:00000001:1.0:1784948878.772731:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948878.772746:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 02000000:00000001:1.0:1784948878.772748:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948878.772757:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948878.772764:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948878.772765:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000007790144d 00000100:00100000:1.0:1784948878.772772:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289659136:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948878.772780:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784948878.772783:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784948878.772787:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:3.0:1784948878.772796:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:1.0:1784948878.772797:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948878.772813:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948878.772818:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007790144d 00000400:00000010:3.0:1784948878.772822:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000007790144d. 02000000:00000010:1.0:1784948878.772827:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000bf7eadde. 02000000:00000001:1.0:1784948878.772833:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.772835:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948878.772844:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:1.0:1784948878.772851:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019017dd3. 00000100:00000040:3.0:1784948878.772852:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000016fc8cf8 x1871651289659008 msgsize 224 00000100:00100000:3.0:1784948878.772869:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000010:1.0:1784948878.772870:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000100:00000040:3.0:1784948878.772872:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000200:1.0:1784948878.772885:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659136, portal 10 00000100:00000001:1.0:1784948878.772893:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948878.772894:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000001:3.0:1784948878.772900:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784948878.772948:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948878.772986:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948878.772998:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948878.772999:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000400:00000010:1.0:1784948878.773020:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 00000100:00000001:2.0:1784948878.773024:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:3.0:1784948878.773027:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00100000:2.0:1784948878.773034:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659008 00000100:00000001:3.0:1784948878.773036:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784948878.773038:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289659136, offset 0 02000000:00000001:2.0:1784948878.773044:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000200:3.0:1784948878.773048:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948878.773053:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948878.773065:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948878.773066:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948878.773088:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.773093:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1784948878.773099:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659008 00000100:00000040:3.0:1784948878.773105:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948878.773107:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1784948878.773108:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000040:2.0:1784948878.773119:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784948878.773135:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784948878.773139:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948878.773140:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948878.773146:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948878.773149:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784948878.773152:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:2.0:1784948878.773160:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784948878.773161:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948878.773172:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:2.0:1784948878.773176:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784948878.773185:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3347:3347:240:4294967295] 192.168.202.137@tcp LPNI seq info [2163:2163:8:4294967295] 00000100:00000001:2.0:1784948878.773192:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784948878.773201:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:2.0:1784948878.773204:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000800:00000200:1.0:1784948878.773217:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948878.773218:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948878.773218:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784948878.773224:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000100:00000001:3.0:1784948878.773231:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305818155 00000020:00000010:2.0:1784948878.773232:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000100:00000010:3.0:1784948878.773241:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000800:00000200:1.0:1784948878.773243:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000010:2.0:1784948878.773252:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a5bcc72e. 00000100:00000001:3.0:1784948878.773261:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000800:00000200:1.0:1784948878.773263:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000010:2.0:1784948878.773267:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b9ccdd8e. 00000100:00000001:3.0:1784948878.773278:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305818155 00000800:00000200:1.0:1784948878.773281:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000020:00000010:2.0:1784948878.773289:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000010:3.0:1784948878.773295:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000099f45de2. 00000100:00000001:3.0:1784948878.773304:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948878.773312:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305818155 00000100:00000040:2.0:1784948878.773314:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000010:3.0:1784948878.773315:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:1.0:1784948878.773319:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.773324:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948878.773327:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:1.0:1784948878.773330:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2075 usecs (3 jiffies) 00000100:00000001:3.0:1784948878.773331:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305818155 00000100:00000001:2.0:1784948878.773333:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948878.773351:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305818116 00000100:00000001:2.0:1784948878.773352:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948878.773353:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000810fb56e. 00000100:00000001:1.0:1784948878.773375:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305818155 00000100:00000001:3.0:1784948878.773377:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948878.773383:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.773398:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305818116 00000100:00000001:3.0:1784948878.773401:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948878.773404:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305818155 00000100:00000001:3.0:1784948878.773410:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.773412:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305818116 00000100:00000001:3.0:1784948878.773417:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.773418:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305818155 00000100:00000001:2.0:1784948878.773420:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948878.773430:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948878.773432:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305818116 00000100:00000001:1.0:1784948878.773459:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305818155 00000100:00000001:1.0:1784948878.773465:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948878.773470:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948878.773471:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.773473:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948878.773480:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948878.773481:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948878.773485:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.773487:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948878.773491:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1784948878.773502:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 725 00000100:00000001:3.0:1784948878.773503:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.773505:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948878.773518:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948878.773518:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.773527:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948878.773530:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742026320 : -115538967525296 : ffff96eafce20850) 00000100:00000001:1.0:1784948878.773536:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948878.773550:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948878.773550:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000016fc8cf8 x1871651289659008/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:539/0 lens 224/0 e 0 to 0 dl 1784948889 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948878.773562:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948878.773571:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948878.773582:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784948878.773582:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948878.773587:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948878.773600:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000800:00000001:3.0:1784948878.773605:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948878.773607:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784948878.773612:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948878.773614:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784948878.773620:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000800:00000001:3.0:1784948878.773624:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948878.773628:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948878.773631:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948878.773644:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0c9b918 00000020:00000010:2.0:1784948878.773644:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000400:00000200:3.0:1784948878.773646:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:0.0:1784948878.773654:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f0c9b918. 00000100:00000001:0.0:1784948878.773664:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:2.0:1784948878.773666:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000016fc8cf8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289659008:12345-0@lo:400: 00000400:00000200:3.0:1784948878.773673:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437700 00000100:00000200:0.0:1784948878.773677:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948878.773687:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659008 00000400:00000200:3.0:1784948878.773700:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 158064 00000020:00000001:2.0:1784948878.773701:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948878.773707:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784948878.773714:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:0.0:1784948878.773718:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948878.773718:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784948878.773723:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948878.773735:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:3.0:1784948878.773736:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948878.773741:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948878.773744:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:3.0:1784948878.773745:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948878.773747:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1784948878.773759:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:3.0:1784948878.773760:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784948878.773767:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948878.773773:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:2.0:1784948878.773775:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:3.0:1784948878.773779:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:2.0:1784948878.773782:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000010:3.0:1784948878.773787:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000095d61ef9. 00000020:00000001:2.0:1784948878.773790:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948878.773795:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000095d61ef9 x1871651289659136 msgsize 224 00000020:00000001:2.0:1784948878.773799:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948878.773807:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00100000:3.0:1784948878.773808:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:2.0:1784948878.773818:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948878.773830:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007afc0d3b. 00000100:00000001:3.0:1784948878.773834:0:337114:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:2.0:1784948878.773839:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948878.773847:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948878.773848:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948878.773855:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948878.773857:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948878.773868:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659008 00010000:00000001:2.0:1784948878.773875:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948878.773887:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000016fc8cf8 x1871651289659008/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:539/0 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948878.773892:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948878.773898:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659136 02000000:00000001:0.0:1784948878.773902:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00010000:00000001:2.0:1784948878.773916:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948878.773923:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948878.773931:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=50 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948878.773937:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948878.773941:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948878.773945:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948878.773949:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659136 00000020:00000001:0.0:1784948878.773953:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948878.773955:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948878.773985:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948878.773990:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948878.773994:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948878.773997:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948878.774002:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948878.774024:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:0.0:1784948878.774027:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948878.774029:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000020:00000010:0.0:1784948878.774036:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000194fc0c0. 00000100:00000001:2.0:1784948878.774042:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:0.0:1784948878.774043:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000003f616a. 00000020:00000010:0.0:1784948878.774051:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:2.0:1784948878.774052:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948878.774059:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000040:0.0:1784948878.774063:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:2.0:1784948878.774068:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784948878.774069:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948878.774074:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:2.0:1784948878.774076:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948878.774078:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948878.774086:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948878.774088:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948878.774093:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000040:2.0:1784948878.774095:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948878.774096:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948878.774106:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000200:2.0:1784948878.774115:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289659008, offset 224 00000100:00000001:0.0:1784948878.774116:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948878.774130:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948878.774133:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948878.774136:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948878.774144:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 701 00000400:00000200:2.0:1784948878.774152:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:0.0:1784948878.774153:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948878.774160:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482524288 : -115544227027328 : ffff96e9c3646680) 00000100:00000040:0.0:1784948878.774176:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000095d61ef9 x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:539/0 lens 224/0 e 0 to 0 dl 1784948889 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948878.774176:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437680 00000400:00000200:2.0:1784948878.774196:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x866d [1] + 224 00000400:00000200:2.0:1784948878.774213:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948878.774224:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948878.774226:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948878.774232:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784948878.774234:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:0.0:1784948878.774241:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784948878.774242:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948878.774249:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00100000:0.0:1784948878.774266:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000095d61ef9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289659136:12345-192.168.202.137@tcp:400: 00000100:00000040:2.0:1784948878.774273:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948878.774279:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659136 00000020:00000001:0.0:1784948878.774286:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948878.774291:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948878.774295:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948878.774302:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948878.774304:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948878.774307:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948878.774311:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948878.774313:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948878.774315:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:2.0:1784948878.774317:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1784948878.774318:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948878.774324:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948878.774328:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948878.774332:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:2.0:1784948878.774332:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 02000000:00000010:0.0:1784948878.774339:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c7c247e1. 00000400:00000010:2.0:1784948878.774339:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 02000000:00000001:0.0:1784948878.774346:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948878.774350:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948878.774351:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948878.774357:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948878.774357:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00010000:00000040:0.0:1784948878.774364:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659136 00010000:00000001:0.0:1784948878.774366:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784948878.774367:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:0.0:1784948878.774372:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000095d61ef9 x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:539/0 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948878.774380:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784948878.774389:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00010000:00000001:0.0:1784948878.774391:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948878.774395:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948878.774395:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948878.774404:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=45 v=5 (1 1 1 1) 00000020:00000001:2.0:1784948878.774404:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784948878.774409:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948878.774412:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948878.774415:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000016fc8cf8 x1871651289659008/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:539/0 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948878.774417:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784948878.774420:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000400:00000001:3.0:1784948878.774428:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784948878.774429:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000001:3.0:1784948878.774434:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1784948878.774437:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:3.0:1784948878.774440:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a51bb92. 02000000:00000001:0.0:1784948878.774443:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948878.774452:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784948878.774453:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000400:00000010:3.0:1784948878.774474:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00100000:2.0:1784948878.774482:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000016fc8cf8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289659008:12345-0@lo:400: Request processed in 849us (1602us total) trans 0 rc 0/0 00000100:00000040:0.0:1784948878.774483:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948878.774486:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000010:0.0:1784948878.774491:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000100:00000200:3.0:1784948878.774500:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948878.774504:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289659136, offset 224 00000400:00000200:0.0:1784948878.774528:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:2.0:1784948878.774530:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 725 00000100:00000200:3.0:1784948878.774531:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948878.774546:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:0.0:1784948878.774552:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948878.774560:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948878.774570:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948878.774571:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948878.774573:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3348:3348:240:4294967295] 192.168.202.137@tcp LPNI seq info [2164:2164:8:4294967295] 00000100:00000001:3.0:1784948878.774579:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948878.774585:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b9ccdd8e. 00000100:00000001:3.0:1784948878.774587:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948878.774594:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948878.774599:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:2.0:1784948878.774603:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784948878.774604:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948878.774611:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:3.0:1784948878.774615:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948878.774619:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 00000100:00001000:3.0:1784948878.774628:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=50 v=5 (5 5 5 5) 00000800:00000200:0.0:1784948878.774630:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000040:2.0:1784948878.774634:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000800:00000010:0.0:1784948878.774643:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000100:00001000:3.0:1784948878.774645:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=50 v=5 (1 1 1 1) 02000000:00000001:2.0:1784948878.774650:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948878.774657:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:2.0:1784948878.774660:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007afc0d3b. 00000800:00000200:0.0:1784948878.774664:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948878.774665:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948878.774671:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:3.0:1784948878.774677:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:2.0:1784948878.774682:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784948878.774683:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948878.774684:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784948878.774697:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948878.774698:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:2.0:1784948878.774701:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:3.0:1784948878.774711:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948878.774717:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000086a2279a. 00000100:00000001:0.0:1784948878.774730:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.774731:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784948878.774734:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b0990f86. 00000100:00000001:3.0:1784948878.774743:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1784948878.774745:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948878.774748:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948878.774754:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948878.774758:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948878.774762:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948878.774764:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948878.774779:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000095d61ef9 x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:539/0 lens 224/224 e 0 to 0 dl 1784948889 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948878.774783:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289659008:0@lo:400: 00000100:00000001:3.0:1784948878.774801:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948878.774807:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1784948878.774813:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000095d61ef9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289659136:12345-192.168.202.137@tcp:400: Request processed in 581us (1010us total) trans 0 rc 0/0 00000100:00000040:3.0:1784948878.774817:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c39699a8 x1871651289659008/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948878.774840:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 701 00000100:00000001:3.0:1784948878.774843:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:0.0:1784948878.774848:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 02000000:00000001:3.0:1784948878.774849:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948878.774855:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000898cc47a. 00000100:00000001:0.0:1784948878.774856:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948878.774865:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:3.0:1784948878.774867:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:0.0:1784948878.774872:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000003f616a. 00000020:00000001:3.0:1784948878.774873:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948878.774880:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000010:0.0:1784948878.774883:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000001:3.0:1784948878.774887:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000010:0.0:1784948878.774893:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000194fc0c0. 02000000:00000010:3.0:1784948878.774894:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f64a2f6c. 00000020:00000040:0.0:1784948878.774903:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 02000000:00000001:3.0:1784948878.774905:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948878.774908:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948878.774910:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c39699a8. 00000100:00000001:3.0:1784948878.774915:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948878.774948:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948878.774953:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948878.774997:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784948878.775000:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784948878.775009:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000100:00000001:3.0:1784948878.775013:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948878.775017:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948878.775021:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948878.775024:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948878.775025:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948878.775034:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948878.775039:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948878.775050:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000400:00000010:1.0:1784948878.775057:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000800:00000001:2.0:1784948878.775059:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948878.775068:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784948878.775069:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948878.775072:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948878.775081:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948878.775098:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948878.775113:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437700 00000400:00000200:2.0:1784948878.775124:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8675 [1] + 224 00000800:00000001:2.0:1784948878.775132:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948878.775140:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948878.775145:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948878.775152:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948878.775158:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948878.775164:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948878.775180:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948878.775209:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948878.775219:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948878.775223:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948878.775257:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948878.775262:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948878.775269:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948878.775272:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948878.775275:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019017dd3. 00000400:00000200:1.0:1784948878.775280:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784948878.775282:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000100:00000001:1.0:1784948878.775286:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948878.775293:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948878.775305:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948878.775315:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948878.775316:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948878.775320:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948878.775324:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948878.775326:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948878.775329:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948878.775332:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948878.775341:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=45 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948878.775346:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=45 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948878.775349:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948878.775351:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948878.775354:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948878.775359:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948878.775368:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.775372:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948878.775374:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948878.775376:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948878.775377:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948878.775381:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948878.775388:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948878.775392:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948878.775394:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948878.775397:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948878.775406:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289659136:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948878.775411:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948878.775413:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948878.775415:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b3c18d2b x1871651289659136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948878.775423:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948878.775426:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948878.775429:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000bf7eadde. 02000000:00000001:1.0:1784948878.775432:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948878.775435:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948878.775437:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948878.775440:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948878.775453:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000000d1fbdb0. 02000000:00000001:1.0:1784948878.775457:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948878.775459:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948878.775460:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b3c18d2b. 00000100:00000001:1.0:1784948878.775464:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948878.775466:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948878.775470:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948878.775477:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948878.775480:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948878.775484:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948878.775487:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948878.816063:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305818155, at 4305818156 00000100:00000040:1.0:1784948878.816079:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305818155, at 4305818156 00000100:00000010:3.0:1784948878.816080:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000810fb56e. 00000100:00000040:0.0:1784948878.816090:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305818155, at 4305818156 00000100:00000040:2.0:1784948878.816091:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305818155, at 4305818156 00000100:00000010:1.0:1784948878.816104:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000099f45de2. 00000100:00000010:2.0:1784948878.816111:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000010:0.0:1784948878.816114:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00010000:00000001:2.0:1784948879.604143:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948879.604156:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948879.604167:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948879.604169:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948879.604295:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948879.604305:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948879.604311:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948879.796019:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948879.796035:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948879.796038:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948879.796045:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948879.796050:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948879.796054:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948879.796057:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948880.628038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948880.628047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948880.628054:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948880.628058:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948880.628060:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948880.628064:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948880.628066:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948880.628069:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948880.628070:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948880.628072:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948880.628132:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948880.628137:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948880.628139:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948880.820023:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948880.820039:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948880.820041:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948880.820047:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948880.820050:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948880.820056:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948880.820059:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948881.652057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948881.652071:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948881.652082:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948881.652084:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948881.652234:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948881.652241:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948881.652246:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948881.844091:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948881.844112:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948881.844116:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948881.844127:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948881.844131:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948881.844137:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948881.844142:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948882.676037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948882.676048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948882.676057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948882.676058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948882.676145:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948882.676151:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948882.676155:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948882.868150:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948882.868168:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948882.868171:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948882.868177:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948882.868181:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948882.868185:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948882.868188:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948883.700037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948883.700049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948883.700060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948883.700062:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948883.700169:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948883.700176:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948883.700180:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1784948883.892129:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948883.892145:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948883.892158:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b8226094. 00000020:00000040:2.0:1784948883.892170:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948883.892175:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948883.892178:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948883.892193:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784948883.892199:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1784948883.892202:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.892204:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948883.892216:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948883.892221:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1784948883.892224:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000662152ce. 00000100:00000001:0.0:1784948883.892226:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.892235:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948883.892240:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948883.892257:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948883.892271:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948883.892275:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948883.892283:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8226094] to pc [ptlrpcd_00_01+1] req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948883.892326:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948883.892329:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948883.892334:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948883.892339:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948883.892342:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948883.892346:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008d623923. 00000020:00000040:2.0:1784948883.892349:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948883.892352:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948883.892354:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948883.892356:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948883.892360:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000820f02b3. 00000100:00000001:2.0:1784948883.892365:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948883.892375:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948883.892384:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948883.892385:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948883.892386:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948883.892397:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000040:2.0:1784948883.892402:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d623923] to pc [ptlrpcd_00_02+2] req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948883.892405:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948883.892421:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948883.892434:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948883.892440:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948883.892441:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.892443:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948883.892452:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948883.892456:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11160) 00000100:00000040:0.0:1784948883.892469:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948883.892476:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948883.892486:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948883.892488:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948883.892494:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948883.892499:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948883.892500:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.892507:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948883.892508:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948883.892509:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948883.892515:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948883.892534:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948883.892541:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1784948883.892543:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948883.892546:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948883.892553:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948883.892555:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948883.892561:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784948883.892562:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289659264:0@lo:400: 02000000:00000001:3.0:1784948883.892571:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.892577:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1784948883.892584:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289659392:192.168.202.137@tcp:400: 02000000:00000001:0.0:1784948883.892590:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948883.892591:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948883.892596:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948883.892600:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1784948883.892600:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948883.892608:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948883.892612:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948883.892617:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000194fc0c0. 02000000:00000010:3.0:1784948883.892620:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000898cc47a. 02000000:00000001:0.0:1784948883.892624:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948883.892626:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948883.892646:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a51bb92. 00000400:00000010:0.0:1784948883.892649:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007679eba0. 00000400:00000010:0.0:1784948883.892678:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000400:00000010:3.0:1784948883.892681:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000100:00000200:0.0:1784948883.892698:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659264, portal 25 00000100:00000001:0.0:1784948883.892703:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948883.892707:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470022400 : -115544239529216 : ffff96e9c2a5a300) 00000100:00000200:3.0:1784948883.892707:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659392, portal 10 00000100:00000001:3.0:1784948883.892720:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948883.892725:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470027776 : -115544239523840 : ffff96e9c2a5b800) 00000100:00000040:0.0:1784948883.892729:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948883.892746:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948883.892755:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948883.892770:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948883.892782:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000100:00000001:3.0:1784948883.892795:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:0.0:1784948883.892797:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289659264, offset 0 00000100:00000040:3.0:1784948883.892806:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948883.892815:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784948883.892818:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00000200:3.0:1784948883.892825:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289659392, offset 0 00000400:00000200:3.0:1784948883.892843:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948883.892844:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948883.892867:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437780 00000400:00000200:0.0:1784948883.892877:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7c7d [2] + 0 00000400:00000200:3.0:1784948883.892881:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000010:0.0:1784948883.892898:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009ef01e88. 00000400:00000200:3.0:1784948883.892901:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948883.892960:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948883.892969:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948883.892974:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948883.892976:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000ba56b0c8 00000400:00000200:3.0:1784948883.892982:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3349:3349:240:4294967295] 192.168.202.137@tcp LPNI seq info [2165:2165:8:4294967295] 00000400:00000200:0.0:1784948883.892990:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948883.893010:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948883.893024:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948883.893035:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948883.893039:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba56b0c8 00000800:00000010:3.0:1784948883.893042:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000400:00000010:0.0:1784948883.893045:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000ba56b0c8. 00000100:00000001:0.0:1784948883.893057:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:3.0:1784948883.893059:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784948883.893062:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948883.893069:0:337119:0:(events.c:337:request_in_callback()) incoming req@0000000038d13d38 x1871651289659264 msgsize 224 00000800:00000200:3.0:1784948883.893085:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:0.0:1784948883.893094:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948883.893101:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:3.0:1784948883.893101:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948883.893129:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.893133:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:3.0:1784948883.893135:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2077 usecs (3 jiffies) 00000100:00000001:3.0:1784948883.893141:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305823236 00000400:00000200:0.0:1784948883.893145:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000100:00000010:3.0:1784948883.893148:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000810fb56e. 00000400:00000010:0.0:1784948883.893150:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000100:00000001:0.0:1784948883.893161:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948883.893165:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000200:0.0:1784948883.893167:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948883.893177:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305823236 00000100:00000010:3.0:1784948883.893189:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000068c34307. 00000100:00000001:0.0:1784948883.893191:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948883.893204:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000040:0.0:1784948883.893208:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948883.893222:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948883.893224:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305823236 00000100:00000001:0.0:1784948883.893229:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:3.0:1784948883.893233:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077930f7d. 00000100:00000001:0.0:1784948883.893240:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948883.893242:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948883.893249:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948883.893260:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659264 00000100:00000001:0.0:1784948883.893262:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305823275 00000100:00000001:3.0:1784948883.893262:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305823236 02000000:00000001:2.0:1784948883.893275:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948883.893285:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948883.893289:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948883.893291:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 02000000:00000001:2.0:1784948883.893298:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.893302:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305823236 00000100:00000001:3.0:1784948883.893304:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00100000:2.0:1784948883.893306:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659264 00000020:00000001:2.0:1784948883.893322:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948883.893328:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784948883.893331:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948883.893333:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305823275 00000020:00000001:2.0:1784948883.893334:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948883.893343:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948883.893352:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948883.893355:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948883.893364:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305823236 00000100:00000001:3.0:1784948883.893365:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948883.893366:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948883.893381:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948883.893385:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948883.893387:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305823275 00000100:00000001:3.0:1784948883.893393:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784948883.893402:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:3.0:1784948883.893403:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948883.893413:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305823236 00000800:00000200:1.0:1784948883.893413:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948883.893415:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948883.893415:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948883.893427:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000010:1.0:1784948883.893429:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000020:00000010:2.0:1784948883.893441:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000001:3.0:1784948883.893444:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.893448:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305823275 00000400:00000200:1.0:1784948883.893450:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948883.893452:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948883.893455:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a5bcc72e. 00000020:00000010:2.0:1784948883.893477:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000aa0c980f. 00000400:00000200:1.0:1784948883.893486:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948883.893488:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305823236 00000020:00000010:2.0:1784948883.893488:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:0.0:1784948883.893507:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948883.893513:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000100:00000001:0.0:1784948883.893515:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948883.893517:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784948883.893518:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000010:1.0:1784948883.893525:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00000001:2.0:1784948883.893540:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948883.893541:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948883.893542:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948883.893543:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948883.893552:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:1.0:1784948883.893561:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948883.893561:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948883.893563:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948883.893564:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948883.893579:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948883.893584:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948883.893600:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784948883.893602:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948883.893607:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948883.893611:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948883.893619:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437800 00000100:00000001:2.0:1784948883.893631:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948883.893634:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948883.893636:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784948883.893640:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 158288 00000100:00000001:1.0:1784948883.893644:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784948883.893651:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 726 00000100:00000001:0.0:1784948883.893653:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:3.0:1784948883.893654:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784948883.893658:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948883.893664:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948883.893666:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742024272 : -115538967527344 : ffff96eafce20050) 00000800:00000001:3.0:1784948883.893670:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948883.893682:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948883.893683:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948883.893690:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000038d13d38 x1871651289659264/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:544/0 lens 224/0 e 0 to 0 dl 1784948894 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948883.893698:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948883.893698:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948883.893722:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948883.893724:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000010:2.0:1784948883.893727:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000200:3.0:1784948883.893728:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784948883.893734:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1784948883.893738:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fa1e143a. 00000020:00000010:2.0:1784948883.893739:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000040:3.0:1784948883.893748:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000fa1e143a x1871651289659392 msgsize 224 00000100:00100000:2.0:1784948883.893753:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000038d13d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289659264:12345-0@lo:400: 00000100:00100000:3.0:1784948883.893765:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:2.0:1784948883.893766:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659264 00000020:00000001:2.0:1784948883.893773:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948883.893778:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948883.893780:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948883.893783:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948883.893785:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784948883.893786:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:2.0:1784948883.893790:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948883.893794:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948883.893797:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:3.0:1784948883.893798:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784948883.893801:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:3.0:1784948883.893805:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948883.893807:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948883.893816:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948883.893820:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948883.893824:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948883.893831:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007afc0d3b. 02000000:00000001:2.0:1784948883.893835:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948883.893839:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948883.893842:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948883.893848:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659264 00000100:00000001:0.0:1784948883.893850:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:2.0:1784948883.893853:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00100000:0.0:1784948883.893856:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659392 00010000:00000200:2.0:1784948883.893863:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000038d13d38 x1871651289659264/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:544/0 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948883.893866:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948883.893875:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948883.893884:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948883.893896:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:0.0:1784948883.893897:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948883.893901:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948883.893960:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659392 00000100:00001000:2.0:1784948883.893964:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=55 v=5 (1 1 1 1) 00000020:00000001:0.0:1784948883.893967:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948883.893973:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:2.0:1784948883.893978:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1784948883.893980:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:2.0:1784948883.893986:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000040:0.0:1784948883.893990:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784948883.893992:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:0.0:1784948883.894002:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:2.0:1784948883.894003:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948883.894009:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948883.894011:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948883.894017:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948883.894019:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784948883.894028:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:0.0:1784948883.894032:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000043fe70d5. 00000400:00000010:2.0:1784948883.894034:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:0.0:1784948883.894046:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784948883.894048:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289659264, offset 224 00000020:00000010:0.0:1784948883.894063:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000043fe70d5. 00000400:00000200:2.0:1784948883.894072:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:0.0:1784948883.894082:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ec8543b3. 00000400:00000200:2.0:1784948883.894096:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784948883.894097:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000003f616a. 00000020:00000010:0.0:1784948883.894106:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:2.0:1784948883.894112:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437780 00000400:00000200:2.0:1784948883.894131:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8689 [1] + 224 00000100:00000040:0.0:1784948883.894134:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948883.894140:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:2.0:1784948883.894141:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948883.894144:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948883.894150:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948883.894151:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948883.894170:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948883.894173:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:2.0:1784948883.894181:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948883.894183:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948883.894188:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948883.894212:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948883.894226:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948883.894240:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948883.894240:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948883.894245:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784948883.894251:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784948883.894260:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00100000:0.0:1784948883.894261:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 702 00000100:00000001:2.0:1784948883.894270:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:0.0:1784948883.894273:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948883.894274:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948883.894282:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482520704 : -115544227030912 : ffff96e9c3645880) 00000100:00000001:2.0:1784948883.894284:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948883.894305:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000fa1e143a x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:544/0 lens 224/0 e 0 to 0 dl 1784948894 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948883.894307:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784948883.894323:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948883.894331:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948883.894334:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948883.894345:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948883.894347:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948883.894361:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000001:1.0:1784948883.894364:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784948883.894369:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000038d13d38 x1871651289659264/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:544/0 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948883.894372:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:0.0:1784948883.894373:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000043fe70d5. 00000400:00000010:1.0:1784948883.894383:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007679eba0. 00000100:00000001:0.0:1784948883.894386:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948883.894399:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000020:00000010:0.0:1784948883.894402:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000043fe70d5. 00000400:00000010:1.0:1784948883.894408:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:1.0:1784948883.894421:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784948883.894438:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000fa1e143a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289659392:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948883.894439:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948883.894439:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000038d13d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289659264:12345-0@lo:400: Request processed in 699us (1351us total) trans 0 rc 0/0 00000100:00000200:0.0:1784948883.894477:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659392 00000020:00000001:0.0:1784948883.894491:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:2.0:1784948883.894493:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 726 00000100:00000200:1.0:1784948883.894502:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948883.894511:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:0.0:1784948883.894512:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784948883.894517:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1784948883.894523:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948883.894525:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948883.894526:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948883.894540:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784948883.894542:0:337119:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:2.0:1784948883.894542:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:0.0:1784948883.894560:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 02000000:00000001:1.0:1784948883.894561:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:2.0:1784948883.894563:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000aa0c980f. 00000100:00000001:1.0:1784948883.894569:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:0.0:1784948883.894575:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784948883.894577:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948883.894579:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000001:0.0:1784948883.894581:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948883.894583:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948883.894594:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 00000020:00000001:0.0:1784948883.894596:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00001000:1.0:1784948883.894599:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=55 v=5 (5 5 5 5) 00000020:00000001:0.0:1784948883.894607:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00001000:1.0:1784948883.894612:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=55 v=5 (1 1 1 1) 00000020:00000040:2.0:1784948883.894612:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:0.0:1784948883.894614:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948883.894621:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948883.894633:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948883.894636:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 02000000:00000001:2.0:1784948883.894639:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:1.0:1784948883.894640:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000010:2.0:1784948883.894644:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007afc0d3b. 00000100:00000001:0.0:1784948883.894649:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00010000:00000001:1.0:1784948883.894651:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948883.894652:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:0.0:1784948883.894659:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948883.894661:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948883.894662:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948883.894670:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006ac2bbf5. 00000100:00000200:2.0:1784948883.894673:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:1.0:1784948883.894679:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948883.894682:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948883.894686:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000529cd956. 00000100:00000001:0.0:1784948883.894701:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948883.894704:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c48eba80. 00000100:00000001:1.0:1784948883.894706:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784948883.894709:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948883.894720:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659392 00000100:00000001:1.0:1784948883.894720:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948883.894728:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948883.894730:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:1.0:1784948883.894742:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:0.0:1784948883.894747:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000fa1e143a x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:544/0 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948883.894836:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289659264:0@lo:400: 00000100:00000001:1.0:1784948883.894857:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948883.894867:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948883.894879:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b8226094 x1871651289659264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948883.894884:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948883.894889:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948883.894957:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948883.894962:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00001000:0.0:1784948883.894966:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=50 v=5 (1 1 1 1) 02000000:00000010:1.0:1784948883.894967:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000194fc0c0. 02000000:00000001:1.0:1784948883.894980:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784948883.894982:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1784948883.894985:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948883.894995:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000040:0.0:1784948883.894997:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000001:1.0:1784948883.895021:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:0.0:1784948883.895023:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000010:1.0:1784948883.895032:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000662152ce. 02000000:00000001:0.0:1784948883.895034:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948883.895040:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948883.895043:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1784948883.895049:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784948883.895050:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948883.895065:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b8226094. 00000100:00000040:0.0:1784948883.895076:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948883.895081:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000400:00000010:0.0:1784948883.895082:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000100:00000001:1.0:1784948883.895084:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948883.895094:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948883.895095:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289659392, offset 224 00000100:00000001:1.0:1784948883.895102:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948883.895107:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948883.895111:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948883.895114:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948883.895119:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948883.895128:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948883.895137:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3350:3350:240:4294967295] 192.168.202.137@tcp LPNI seq info [2166:2166:8:4294967295] 00000400:00000200:0.0:1784948883.895154:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948883.895161:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948883.895167:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000595fa8f8. 00000800:00000200:0.0:1784948883.895176:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948883.895186:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948883.895191:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000595fa8f8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948883.895211:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948883.895215:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948883.895218:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948883.895220:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948883.895223:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948883.895227:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fa1e143a x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:544/0 lens 224/224 e 0 to 0 dl 1784948894 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948883.895240:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000fa1e143a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289659392:12345-192.168.202.137@tcp:400: Request processed in 880us (1490us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948883.895250:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 702 00000100:00000040:0.0:1784948883.895254:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948883.895257:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948883.895259:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948883.895264:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000003f616a. 00000020:00000010:0.0:1784948883.895270:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948883.895275:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000ec8543b3. 00000020:00000040:0.0:1784948883.895285:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948883.895290:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948883.895407:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948883.895413:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000595fa8f8. 00000400:00000200:2.0:1784948883.895419:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948883.895427:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948883.895434:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000800:00000001:1.0:1784948883.895435:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784948883.895437:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000100:00000001:2.0:1784948883.895443:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948883.895446:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784948883.895447:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948883.895452:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948883.895457:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948883.895465:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948883.895478:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437800 00000400:00000200:1.0:1784948883.895484:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x868d [1] + 224 00000800:00000001:1.0:1784948883.895490:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948883.895497:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948883.895499:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948883.895503:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948883.895508:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948883.895511:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948883.895521:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948883.895541:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948883.895547:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948883.895549:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948883.895620:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948883.895625:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948883.895630:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948883.895632:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948883.895634:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a51bb92. 00000400:00000200:3.0:1784948883.895642:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000400:00000010:3.0:1784948883.895643:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:3.0:1784948883.895648:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948883.895653:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948883.895663:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948883.895671:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948883.895673:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948883.895675:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948883.895677:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948883.895678:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948883.895680:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948883.895682:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948883.895688:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=50 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948883.895691:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=50 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948883.895693:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948883.895694:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948883.895696:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948883.895700:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948883.895707:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948883.895710:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948883.895712:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948883.895713:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948883.895714:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948883.895717:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948883.895723:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948883.895725:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948883.895727:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948883.895729:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948883.895736:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289659392:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948883.895741:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948883.895743:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948883.895745:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008d623923 x1871651289659392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948899 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948883.895751:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948883.895752:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948883.895754:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000898cc47a. 02000000:00000001:3.0:1784948883.895757:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948883.895758:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948883.895760:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948883.895762:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948883.895763:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000820f02b3. 02000000:00000001:3.0:1784948883.895766:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948883.895767:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948883.895769:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008d623923. 00000100:00000001:3.0:1784948883.895771:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948883.895773:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948883.895774:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948883.895778:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948883.895779:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948883.895781:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948883.895783:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948883.936054:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305823275, at 4305823276 00000100:00000040:2.0:1784948883.936065:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305823275, at 4305823276 00000100:00000010:1.0:1784948883.936072:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000068c34307. 00000100:00000010:2.0:1784948883.936081:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077930f7d. 00000100:00000040:3.0:1784948883.936103:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305823275, at 4305823276 00000100:00000040:0.0:1784948883.936106:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305823275, at 4305823276 00000100:00000010:3.0:1784948883.936128:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000010:0.0:1784948883.936134:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000810fb56e. 00010000:00000001:2.0:1784948884.724039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948884.724050:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948884.724059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948884.724061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948884.724182:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948884.724188:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948884.724192:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948884.916116:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948884.916139:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948884.916142:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948884.916149:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948884.916154:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948884.916159:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948884.916163:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948885.748042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948885.748051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948885.748058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948885.748059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948885.748157:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948885.748161:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948885.748164:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948885.940014:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948885.940028:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948885.940030:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948885.940035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948885.940038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948885.940043:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948885.940046:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948886.772087:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948886.772100:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948886.772111:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948886.772113:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948886.772220:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948886.772228:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948886.772233:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948886.964053:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948886.964070:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948886.964073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948886.964083:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948886.964087:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948886.964092:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948886.964096:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948887.796096:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948887.796109:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948887.796119:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948887.796122:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948887.796204:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948887.796214:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948887.796219:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948887.988048:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948887.988066:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948887.988069:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948887.988075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948887.988081:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948887.988086:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948887.988089:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948888.820044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948888.820054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948888.820061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948888.820063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948888.820128:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948888.820133:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948888.820137:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1784948889.012026:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948889.012041:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948889.012056:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000078cad088. 00000020:00000040:2.0:1784948889.012066:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948889.012072:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948889.012074:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948889.012079:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948889.012088:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1784948889.012089:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948889.012095:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948889.012106:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1784948889.012107:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000820f02b3. 00000100:00000001:1.0:1784948889.012111:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948889.012119:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948889.012123:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948889.012125:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948889.012157:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948889.012173:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948889.012177:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948889.012185:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000078cad088] to pc [ptlrpcd_00_03+3] req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948889.012222:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948889.012226:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948889.012231:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948889.012235:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948889.012239:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948889.012243:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086d6bb31. 00000020:00000040:2.0:1784948889.012245:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948889.012247:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948889.012248:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948889.012250:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784948889.012251:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948889.012260:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000662152ce. 00000100:00000001:3.0:1784948889.012260:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948889.012266:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.012267:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948889.012282:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1784948889.012282:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.012308:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948889.012310:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948889.012317:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.012323:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948889.012326:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086d6bb31] to pc [ptlrpcd_00_00+0] req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948889.012326:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948889.012336:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948889.012349:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948889.012357:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948889.012363:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948889.012368:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948889.012369:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289659520:0@lo:400: 00000100:00000040:2.0:1784948889.012376:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11166) 00000100:00000001:3.0:1784948889.012377:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948889.012388:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948889.012393:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.012396:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948889.012398:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1784948889.012410:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 02000000:00000010:3.0:1784948889.012413:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000898cc47a. 00000100:00000001:1.0:1784948889.012421:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948889.012423:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.012424:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948889.012430:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948889.012441:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a51bb92. 00000100:00000001:0.0:1784948889.012443:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948889.012456:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.012461:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948889.012463:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:3.0:1784948889.012463:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000100:00000001:0.0:1784948889.012470:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000200:3.0:1784948889.012478:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659520, portal 25 00000100:00000040:0.0:1784948889.012489:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948889.012491:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948889.012497:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470018816 : -115544239532800 : ffff96e9c2a59500) 00000100:00000040:3.0:1784948889.012521:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.012541:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948889.012549:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.012563:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784948889.012564:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948889.012576:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948889.012586:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784948889.012598:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948889.012599:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000100:00000200:3.0:1784948889.012610:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289659520, offset 0 00000100:00100000:0.0:1784948889.012617:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289659648:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784948889.012622:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948889.012637:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948889.012648:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:3.0:1784948889.012652:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:0.0:1784948889.012655:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948889.012665:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784948889.012672:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437880 02000000:00000010:0.0:1784948889.012677:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000ec8543b3. 00000400:00000200:3.0:1784948889.012684:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7c9d [2] + 0 02000000:00000001:0.0:1784948889.012694:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948889.012696:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d97ec0aa. 00000400:00000200:3.0:1784948889.012716:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000007e436012 00000400:00000010:0.0:1784948889.012717:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ee18875c. 00000400:00000010:0.0:1784948889.012734:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000400:00000200:3.0:1784948889.012735:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948889.012748:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:0.0:1784948889.012753:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659648, portal 10 00000400:00000200:3.0:1784948889.012757:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007e436012 00000100:00000001:0.0:1784948889.012765:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:3.0:1784948889.012766:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000007e436012. 00000100:00000001:0.0:1784948889.012776:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470019712 : -115544239531904 : ffff96e9c2a59880) 00000100:00000001:3.0:1784948889.012781:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948889.012786:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784948889.012792:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000043c318de x1871651289659520 msgsize 224 00000100:00100000:3.0:1784948889.012800:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948889.012807:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948889.012808:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784948889.012828:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948889.012831:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784948889.012843:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000100:00000040:0.0:1784948889.012846:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948889.012848:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000400:00000010:0.0:1784948889.012862:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000236111b7. 00000100:00000001:3.0:1784948889.012862:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784948889.012933:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948889.012940:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948889.012947:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659520 02000000:00000001:2.0:1784948889.012952:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948889.012953:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948889.012956:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784948889.012961:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948889.012962:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948889.012967:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289659648, offset 0 02000000:00000001:2.0:1784948889.012976:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948889.012989:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659520 00000100:00000001:3.0:1784948889.012992:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948889.012996:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948889.013000:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:2.0:1784948889.013003:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:0.0:1784948889.013005:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784948889.013013:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948889.013016:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948889.013019:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948889.013025:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305828395 00000020:00000001:2.0:1784948889.013026:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784948889.013037:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:2.0:1784948889.013038:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784948889.013039:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00000001:2.0:1784948889.013061:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948889.013061:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000200:0.0:1784948889.013064:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:2.0:1784948889.013072:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:2.0:1784948889.013080:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784948889.013081:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948889.013083:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305828395 00000020:00000010:2.0:1784948889.013099:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00000010:3.0:1784948889.013100:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007024d30a. 00000400:00000200:0.0:1784948889.013106:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:2.0:1784948889.013117:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000100:00000001:3.0:1784948889.013117:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000400:00000200:0.0:1784948889.013122:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784948889.013133:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305828395 00000020:00000010:2.0:1784948889.013140:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000aa0c980f. 00000100:00000010:3.0:1784948889.013145:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000810fb56e. 00000400:00000200:0.0:1784948889.013147:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3351:3351:240:4294967295] 192.168.202.137@tcp LPNI seq info [2167:2167:8:4294967295] 00000020:00000010:2.0:1784948889.013155:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784948889.013160:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000040:2.0:1784948889.013181:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948889.013184:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305828395 00000100:00000001:2.0:1784948889.013187:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948889.013200:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000010:3.0:1784948889.013200:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077930f7d. 00000100:00000001:2.0:1784948889.013207:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.013213:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000200:0.0:1784948889.013216:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784948889.013226:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.013228:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948889.013241:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.013245:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948889.013252:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948889.013253:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948889.013258:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948889.013267:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948889.013271:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948889.013278:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:0.0:1784948889.013279:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b84187a1. 00000100:00000001:2.0:1784948889.013284:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948889.013285:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948889.013294:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948889.013296:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948889.013306:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948889.013314:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 727 00000800:00000200:0.0:1784948889.013320:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:2.0:1784948889.013326:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948889.013332:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742028368 : -115538967523248 : ffff96eafce21050) 00000100:00000040:2.0:1784948889.013344:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000043c318de x1871651289659520/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:550/0 lens 224/0 e 0 to 0 dl 1784948900 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948889.013359:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784948889.013375:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:0.0:1784948889.013377:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b84187a1 type 1, nob 320 niov 1 nkiov 1 00000020:00000010:2.0:1784948889.013384:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:2.0:1784948889.013390:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948889.013394:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00100000:2.0:1784948889.013400:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000043c318de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289659520:12345-0@lo:400: 00000100:00000200:2.0:1784948889.013404:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659520 00000020:00000001:2.0:1784948889.013408:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948889.013415:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948889.013416:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948889.013423:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948889.013429:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948889.013434:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000040:0.0:1784948889.013435:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2080 usecs (3 jiffies) 00000020:00000001:2.0:1784948889.013446:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784948889.013453:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305828356 00000020:00000001:2.0:1784948889.013454:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948889.013466:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784948889.013471:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305828395 00000020:00000001:2.0:1784948889.013471:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948889.013480:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.013492:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305828356 00000020:00000001:2.0:1784948889.013493:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948889.013503:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948889.013506:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305828395 02000000:00000001:2.0:1784948889.013509:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948889.013517:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c4a1e5eb. 00000100:00000001:0.0:1784948889.013519:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305828356 02000000:00000001:2.0:1784948889.013524:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.013531:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305828395 00000100:00000001:2.0:1784948889.013532:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948889.013538:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.013545:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305828356 00010000:00000040:2.0:1784948889.013547:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659520 00010000:00000001:2.0:1784948889.013554:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948889.013564:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305828395 00010000:00000200:2.0:1784948889.013566:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000043c318de x1871651289659520/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:550/0 lens 224/224 e 0 to 0 dl 1784948900 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.013580:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784948889.013586:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948889.013590:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948889.013591:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948889.013606:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=61 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948889.013610:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948889.013612:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948889.013614:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1784948889.013627:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b84187a1. 00000100:00000001:0.0:1784948889.013632:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948889.013632:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:3.0:1784948889.013635:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948889.013643:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:1.0:1784948889.013646:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784948889.013653:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948889.013657:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948889.013663:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948889.013664:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:1.0:1784948889.013672:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948889.013675:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:3.0:1784948889.013675:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000236111b7 00000400:00000010:3.0:1784948889.013690:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000236111b7. 00000800:00000001:1.0:1784948889.013692:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784948889.013695:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:3.0:1784948889.013706:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948889.013708:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289659520, offset 224 00000800:00000001:1.0:1784948889.013711:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784948889.013720:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948889.013721:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948889.013742:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948889.013751:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784948889.013777:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948889.013786:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437900 00000100:00000040:3.0:1784948889.013788:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948889.013797:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437880 00000100:00000001:3.0:1784948889.013822:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948889.013833:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 158512 00000100:00000001:3.0:1784948889.013835:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784948889.013855:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x86a5 [1] + 224 00000800:00000001:1.0:1784948889.013857:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948889.013888:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948889.013892:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948889.013897:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:1.0:1784948889.013899:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948889.013954:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784948889.013959:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948889.013969:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948889.013989:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:2.0:1784948889.013995:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948889.013997:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948889.014015:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005f948c2e. 00000100:00000040:1.0:1784948889.014022:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000005f948c2e x1871651289659648 msgsize 224 00000100:00000001:2.0:1784948889.014025:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1784948889.014038:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784948889.014042:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784948889.014047:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00000001:2.0:1784948889.014052:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948889.014054:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948889.014059:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948889.014065:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784948889.014066:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948889.014068:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1784948889.014070:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784948889.014073:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948889.014075:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948889.014080:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:3.0:1784948889.014084:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:1.0:1784948889.014086:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784948889.014088:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000043c318de x1871651289659520/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:550/0 lens 224/224 e 0 to 0 dl 1784948900 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784948889.014095:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000001:1.0:1784948889.014100:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948889.014102:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a51bb92. 00000400:00000200:3.0:1784948889.014123:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000400:00000010:3.0:1784948889.014129:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00100000:2.0:1784948889.014132:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000043c318de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289659520:12345-0@lo:400: Request processed in 747us (1330us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948889.014137:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948889.014145:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:3.0:1784948889.014147:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948889.014151:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 727 00000100:00100000:0.0:1784948889.014159:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659648 00000100:00000040:2.0:1784948889.014163:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784948889.014175:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:0.0:1784948889.014176:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948889.014180:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:3.0:1784948889.014180:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.014186:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:2.0:1784948889.014189:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000aa0c980f. 00000100:00000001:0.0:1784948889.014198:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948889.014202:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784948889.014204:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948889.014214:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948889.014215:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 02000000:00000001:0.0:1784948889.014217:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.014221:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000040:2.0:1784948889.014226:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:3.0:1784948889.014230:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948889.014238:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:0.0:1784948889.014239:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659648 02000000:00000001:2.0:1784948889.014239:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948889.014242:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948889.014246:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c4a1e5eb. 02000000:00000001:3.0:1784948889.014253:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948889.014258:0:342762:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1784948889.014259:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:3.0:1784948889.014262:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=61 v=5 (5 5 5 5) 00000100:00000001:2.0:1784948889.014265:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784948889.014270:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000200:2.0:1784948889.014275:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:3.0:1784948889.014275:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=61 v=5 (1 1 1 1) 00000020:00000001:0.0:1784948889.014279:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:2.0:1784948889.014285:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008c614621. 00000100:00000001:3.0:1784948889.014291:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000040:0.0:1784948889.014292:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000010:2.0:1784948889.014293:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fe7b2a7d. 00000100:00000001:3.0:1784948889.014303:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948889.014307:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:3.0:1784948889.014312:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:0.0:1784948889.014319:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:3.0:1784948889.014320:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.014330:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948889.014332:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948889.014342:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000025dbf3d7. 00000100:00000040:3.0:1784948889.014348:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.014352:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948889.014367:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000025dbf3d7. 00000100:00000001:3.0:1784948889.014383:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948889.014387:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:0.0:1784948889.014389:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b8810b40. 00000100:00000001:3.0:1784948889.014390:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948889.014401:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948889.014408:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f57325dd. 00000100:00100000:3.0:1784948889.014414:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289659520:0@lo:400: 00000020:00000010:0.0:1784948889.014423:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784948889.014425:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948889.014436:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948889.014439:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000078cad088 x1871651289659520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948889.014450:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948889.014451:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948889.014457:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948889.014464:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000898cc47a. 00000100:00000001:0.0:1784948889.014466:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:3.0:1784948889.014470:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784948889.014475:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:3.0:1784948889.014476:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948889.014488:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948889.014490:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948889.014492:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948889.014497:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000820f02b3. 02000000:00000001:3.0:1784948889.014505:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948889.014507:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1784948889.014509:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000010:3.0:1784948889.014510:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000078cad088. 00000100:00000001:0.0:1784948889.014525:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948889.014526:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948889.014534:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.014536:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.014539:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948889.014553:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948889.014556:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948889.014561:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948889.014563:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.014575:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.014598:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948889.014602:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948889.014613:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 703 00000100:00000040:0.0:1784948889.014620:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948889.014624:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549150848 : -115544160400768 : ffff96e9c75d0a80) 00000100:00000040:0.0:1784948889.014635:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005f948c2e x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:550/0 lens 224/0 e 0 to 0 dl 1784948900 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.014652:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948889.014659:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000025dbf3d7. 00000100:00000001:0.0:1784948889.014663:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948889.014668:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000025dbf3d7. 00000100:00100000:0.0:1784948889.014678:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005f948c2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289659648:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948889.014685:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659648 00000020:00000001:0.0:1784948889.014692:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948889.014699:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948889.014703:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948889.014707:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948889.014711:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948889.014717:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948889.014720:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948889.014725:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948889.014728:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948889.014733:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948889.014736:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948889.014743:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948889.014747:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948889.014754:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a868d044. 02000000:00000001:0.0:1784948889.014760:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.014763:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948889.014767:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948889.014772:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659648 00010000:00000001:0.0:1784948889.014777:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948889.014784:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005f948c2e x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:550/0 lens 224/224 e 0 to 0 dl 1784948900 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948889.014800:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948889.014804:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948889.014810:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=56 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948889.014818:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948889.014823:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948889.014827:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948889.014831:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948889.014834:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.014840:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948889.014844:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948889.014850:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000100:00000200:0.0:1784948889.014857:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289659648, offset 224 00000400:00000200:0.0:1784948889.014866:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948889.014944:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948889.014955:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3352:3352:240:4294967295] 192.168.202.137@tcp LPNI seq info [2168:2168:8:4294967295] 00000400:00000200:0.0:1784948889.014972:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948889.014981:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948889.014989:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a3cf4688. 00000800:00000200:0.0:1784948889.014998:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948889.015009:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948889.015014:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a3cf4688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948889.015037:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948889.015044:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948889.015048:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948889.015053:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948889.015057:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948889.015063:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005f948c2e x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:550/0 lens 224/224 e 0 to 0 dl 1784948900 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948889.015082:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005f948c2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289659648:12345-192.168.202.137@tcp:400: Request processed in 428us (1057us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948889.015096:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 703 00000100:00000040:0.0:1784948889.015102:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948889.015107:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948889.015111:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948889.015119:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f57325dd. 00000020:00000010:0.0:1784948889.015125:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948889.015131:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b8810b40. 00000020:00000040:0.0:1784948889.015138:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948889.015142:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948889.015314:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948889.015325:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a3cf4688. 00000400:00000200:1.0:1784948889.015335:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948889.015339:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948889.015351:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948889.015360:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948889.015362:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000800:00000001:3.0:1784948889.015368:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784948889.015373:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000800:00000001:3.0:1784948889.015375:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948889.015390:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:3.0:1784948889.015393:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784948889.015399:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948889.015409:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437900 00000400:00000200:3.0:1784948889.015420:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x86ad [1] + 224 00000800:00000001:3.0:1784948889.015426:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948889.015437:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948889.015440:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948889.015445:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948889.015450:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948889.015454:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948889.015463:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948889.015482:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948889.015488:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948889.015491:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.015556:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948889.015561:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948889.015569:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948889.015572:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948889.015575:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ee18875c. 00000400:00000200:0.0:1784948889.015582:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784948889.015584:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784948889.015587:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948889.015595:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948889.015607:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.015616:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948889.015617:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.015620:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948889.015624:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948889.015627:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948889.015629:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948889.015633:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948889.015640:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=56 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948889.015644:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=56 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948889.015648:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948889.015650:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948889.015654:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948889.015661:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948889.015676:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.015681:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948889.015684:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948889.015688:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948889.015690:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948889.015697:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.015711:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948889.015716:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948889.015719:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948889.015725:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948889.015740:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289659648:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948889.015750:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.015754:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948889.015758:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000086d6bb31 x1871651289659648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948889.015772:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948889.015776:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948889.015780:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000ec8543b3. 02000000:00000001:0.0:1784948889.015792:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948889.015796:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948889.015798:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948889.015803:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948889.015807:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000662152ce. 02000000:00000001:0.0:1784948889.015813:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948889.015816:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948889.015818:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000086d6bb31. 00000100:00000001:0.0:1784948889.015823:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948889.015825:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.015827:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948889.015833:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948889.015835:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948889.015837:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948889.015839:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948889.056007:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305828395, at 4305828396 00000100:00000010:3.0:1784948889.056023:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077930f7d. 00000100:00000040:2.0:1784948889.056030:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305828395, at 4305828396 00000100:00000010:2.0:1784948889.056048:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000810fb56e. 00000100:00000040:1.0:1784948889.056078:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305828395, at 4305828396 00000100:00000040:0.0:1784948889.056083:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305828395, at 4305828396 00000100:00000010:1.0:1784948889.056101:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007024d30a. 00000100:00000010:0.0:1784948889.056106:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00010000:00000001:2.0:1784948889.844064:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948889.844076:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948889.844084:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948889.844086:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948889.844188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948889.844200:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948889.844207:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948890.036069:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948890.036084:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948890.036087:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948890.036093:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948890.036097:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948890.036101:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948890.036105:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948890.868049:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948890.868061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948890.868071:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948890.868076:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948890.868079:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948890.868083:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948890.868087:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948890.868089:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948890.868092:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948890.868094:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948890.868172:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948890.868179:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948890.868185:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948891.060115:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948891.060133:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948891.060136:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948891.060144:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948891.060148:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948891.060152:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948891.060156:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948891.892095:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948891.892111:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948891.892126:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948891.892130:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948891.892249:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948891.892259:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948891.892266:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948892.083981:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948892.083997:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948892.083999:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948892.084003:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948892.084006:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948892.084009:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948892.084012:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948892.916107:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948892.916120:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948892.916131:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948892.916133:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948892.916278:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948892.916287:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948892.916293:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948893.108036:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948893.108058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948893.108063:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948893.108072:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948893.108077:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948893.108083:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948893.108088:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948893.940143:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948893.940157:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948893.940168:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948893.940170:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948893.940276:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948893.940288:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948893.940295:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948894.132074:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948894.132094:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948894.132098:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.132106:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948894.132111:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948894.132111:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948894.132119:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948894.132124:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948894.132125:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948894.132135:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b16a92e. 00000020:00000040:3.0:1784948894.132145:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948894.132152:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948894.132156:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948894.132163:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948894.132172:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000001:3.0:1784948894.132181:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948894.132192:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948894.132213:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948894.132217:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948894.132225:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b16a92e] to pc [ptlrpcd_00_01+1] req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948894.132272:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948894.132278:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948894.132286:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784948894.132292:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948894.132299:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948894.132302:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1784948894.132306:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005dccb977. 00000100:00000001:0.0:1784948894.132310:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948894.132318:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1784948894.132318:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948894.132329:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1784948894.132331:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948894.132337:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948894.132342:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948894.132345:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948894.132353:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948894.132361:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:0.0:1784948894.132370:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948894.132375:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948894.132376:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948894.132387:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948894.132395:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948894.132395:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1784948894.132405:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948894.132419:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289659776:0@lo:400: 00000100:00000001:3.0:1784948894.132425:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948894.132429:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948894.132430:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948894.132439:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784948894.132440:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005dccb977] to pc [ptlrpcd_00_02+2] req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948894.132445:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948894.132454:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948894.132465:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000b8810b40. 02000000:00000001:0.0:1784948894.132472:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948894.132480:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ee18875c. 00000400:00000010:0.0:1784948894.132488:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000001:3.0:1784948894.132490:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948894.132496:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659776, portal 25 00000100:00000040:3.0:1784948894.132499:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11171) 00000100:00000001:0.0:1784948894.132505:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948894.132510:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482521600 : -115544227030016 : ffff96e9c3645c00) 00000100:00000040:0.0:1784948894.132519:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948894.132532:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948894.132539:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948894.132543:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000001:1.0:1784948894.132545:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784948894.132548:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289659776, offset 0 00000100:00000001:2.0:1784948894.132552:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948894.132555:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948894.132565:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948894.132568:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948894.132569:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948894.132594:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948894.132596:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948894.132618:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437980 00000100:00000001:1.0:1784948894.132627:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784948894.132634:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7cc1 [2] + 0 00000100:00000001:1.0:1784948894.132636:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948894.132644:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000af4b2538. 02000000:00000001:1.0:1784948894.132649:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784948894.132654:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000573e919c 02000000:00000001:1.0:1784948894.132656:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948894.132665:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948894.132668:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:1.0:1784948894.132675:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289659904:192.168.202.137@tcp:400: 00000400:00000200:0.0:1784948894.132685:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948894.132693:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:0.0:1784948894.132701:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000573e919c 02000000:00000001:1.0:1784948894.132703:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:0.0:1784948894.132712:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000573e919c. 02000000:00000001:1.0:1784948894.132713:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948894.132722:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948894.132729:0:337120:0:(events.c:286:request_in_callback()) Process entered 02000000:00000010:1.0:1784948894.132733:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000043d62367. 00000100:00000200:0.0:1784948894.132737:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:1.0:1784948894.132742:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948894.132747:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000cfaa2b5b x1871651289659776 msgsize 224 00000400:00000010:1.0:1784948894.132755:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019017dd3. 00000100:00100000:0.0:1784948894.132768:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000010:1.0:1784948894.132773:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000100:00000040:0.0:1784948894.132774:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000200:1.0:1784948894.132787:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289659904, portal 10 00000100:00000001:1.0:1784948894.132793:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948894.132795:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482519808 : -115544227031808 : ffff96e9c3645500) 00000100:00000001:0.0:1784948894.132806:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784948894.132811:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948894.132821:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784948894.132853:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784948894.132868:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948894.132880:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948894.132881:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948894.132885:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1784948894.132890:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659776 02000000:00000001:2.0:1784948894.132900:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948894.132953:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948894.132956:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948894.132972:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948894.132975:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948894.132979:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:1.0:1784948894.132983:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000100:00100000:2.0:1784948894.132985:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659776 00000100:00000040:0.0:1784948894.132993:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948894.132995:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:1.0:1784948894.132998:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289659904, offset 0 00000020:00000040:2.0:1784948894.133006:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784948894.133019:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784948894.133030:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948894.133031:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948894.133033:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948894.133042:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1784948894.133053:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784948894.133063:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:2.0:1784948894.133065:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784948894.133077:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784948894.133081:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784948894.133090:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:2.0:1784948894.133092:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000002fef99f7. 00000100:00000001:2.0:1784948894.133101:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948894.133107:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:2.0:1784948894.133108:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000002fef99f7. 00000400:00000200:1.0:1784948894.133121:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:2.0:1784948894.133123:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000400:00000200:1.0:1784948894.133133:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3353:3353:240:4294967295] 192.168.202.137@tcp LPNI seq info [2169:2169:8:4294967295] 00000020:00000010:2.0:1784948894.133134:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009242fa2b. 00000020:00000010:2.0:1784948894.133147:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000040:2.0:1784948894.133169:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948894.133178:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948894.133181:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948894.133188:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948894.133192:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784948894.133202:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948894.133211:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948894.133213:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948894.133221:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305833515 00000800:00000010:1.0:1784948894.133226:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000100:00000001:2.0:1784948894.133231:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1784948894.133232:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:0.0:1784948894.133245:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000800:00000200:1.0:1784948894.133251:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784948894.133264:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305833515 00000100:00000010:0.0:1784948894.133273:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:2.0:1784948894.133274:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948894.133287:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948894.133289:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000800:00000200:1.0:1784948894.133289:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784948894.133301:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000200:1.0:1784948894.133306:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948894.133307:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305833515 00000100:00100000:2.0:1784948894.133313:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 728 00000100:00000001:3.0:1784948894.133315:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000010:0.0:1784948894.133324:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000040:2.0:1784948894.133336:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948894.133338:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948894.133338:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948894.133349:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742035536 : -115538967516080 : ffff96eafce22c50) 00000100:00000001:3.0:1784948894.133350:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948894.133355:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305833515 00000100:00000001:1.0:1784948894.133358:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948894.133371:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000040:1.0:1784948894.133374:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2083 usecs (3 jiffies) 00000100:00000040:2.0:1784948894.133379:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000cfaa2b5b x1871651289659776/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:555/0 lens 224/0 e 0 to 0 dl 1784948905 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948894.133384:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948894.133393:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305833476 00000100:00000001:0.0:1784948894.133396:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948894.133417:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948894.133418:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305833515 00000100:00000001:2.0:1784948894.133423:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948894.133431:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948894.133433:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000002fef99f7. 00000100:00000001:1.0:1784948894.133434:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305833476 00000100:00000001:0.0:1784948894.133445:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948894.133448:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948894.133449:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305833515 00000020:00000010:2.0:1784948894.133459:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000002fef99f7. 00000100:00000001:0.0:1784948894.133466:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948894.133467:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305833476 00000100:00000001:0.0:1784948894.133475:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1784948894.133476:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000cfaa2b5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289659776:12345-0@lo:400: 00000100:00000001:1.0:1784948894.133484:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305833515 00000100:00000001:0.0:1784948894.133492:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784948894.133504:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659776 00000100:00000001:1.0:1784948894.133506:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305833476 00000100:00000001:0.0:1784948894.133507:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:3.0:1784948894.133513:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948894.133517:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948894.133519:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784948894.133523:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305833515 00000800:00000010:3.0:1784948894.133524:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000020:00000001:2.0:1784948894.133534:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784948894.133535:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948894.133538:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948894.133540:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784948894.133545:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948894.133559:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784948894.133561:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948894.133562:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784948894.133572:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784948894.133580:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948894.133583:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:3.0:1784948894.133584:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000800:00000001:0.0:1784948894.133585:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948894.133590:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948894.133601:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948894.133601:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000100:00000001:1.0:1784948894.133608:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000001:0.0:1784948894.133613:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948894.133627:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:0.0:1784948894.133629:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948894.133629:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948894.133629:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1784948894.133641:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:0.0:1784948894.133647:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.133649:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948894.133649:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948894.133651:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948894.133663:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948894.133668:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948894.133675:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948894.133678:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948894.133691:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948894.133693:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948894.133698:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948894.133700:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1784948894.133702:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c4a1e5eb. 02000000:00000001:2.0:1784948894.133710:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948894.133715:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948894.133720:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948894.133735:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948894.133743:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437a00 00010000:00000040:2.0:1784948894.133748:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659776 00010000:00000001:2.0:1784948894.133767:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948894.133769:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:2.0:1784948894.133779:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000cfaa2b5b x1871651289659776/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:555/0 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948894.133780:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948894.133786:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 158736 00000800:00000001:0.0:1784948894.133825:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784948894.133825:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948894.133852:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948894.133859:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=66 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948894.133868:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:0.0:1784948894.133872:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948894.133873:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948894.133889:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:0.0:1784948894.133890:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784948894.133936:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948894.133939:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948894.133945:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948894.133950:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:0.0:1784948894.133957:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:2.0:1784948894.133959:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:0.0:1784948894.133966:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948894.133970:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289659776, offset 224 00000100:00000200:0.0:1784948894.133971:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784948894.133989:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009778919c. 00000400:00000200:2.0:1784948894.133997:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:0.0:1784948894.134000:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000009778919c x1871651289659904 msgsize 224 00000100:00100000:0.0:1784948894.134009:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784948894.134020:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948894.134025:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784948894.134042:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437980 00000800:00000001:0.0:1784948894.134044:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948894.134054:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948894.134066:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x86c1 [1] + 224 00000400:00000200:2.0:1784948894.134082:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948894.134088:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784948894.134095:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00100000:3.0:1784948894.134096:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289659904 02000000:00000001:3.0:1784948894.134105:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948894.134110:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784948894.134111:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948894.134119:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948894.134124:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948894.134128:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948894.134140:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289659904 00000020:00000001:3.0:1784948894.134148:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:2.0:1784948894.134149:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784948894.134155:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784948894.134166:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948894.134176:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948894.134185:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784948894.134188:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948894.134193:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948894.134195:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784948894.134200:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000001:3.0:1784948894.134205:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948894.134209:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000400:00000200:2.0:1784948894.134211:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784948894.134215:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000020:00000010:3.0:1784948894.134218:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000898cc47a. 00000100:00000001:2.0:1784948894.134221:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948894.134225:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000010:3.0:1784948894.134227:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000aa7a2a20. 00000100:00000001:2.0:1784948894.134233:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948894.134236:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003c019a34. 00000100:00000040:2.0:1784948894.134243:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784948894.134249:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948894.134253:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948894.134254:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948894.134256:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948894.134260:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948894.134260:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784948894.134263:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948894.134271:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:2.0:1784948894.134272:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cfaa2b5b x1871651289659776/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:555/0 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784948894.134275:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784948894.134284:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:0.0:1784948894.134285:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948894.134295:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ee18875c. 00000100:00000001:3.0:1784948894.134303:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:0.0:1784948894.134313:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000100:00000001:3.0:1784948894.134315:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948894.134323:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948894.134326:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00100000:2.0:1784948894.134328:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000cfaa2b5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289659776:12345-0@lo:400: Request processed in 896us (1571us total) trans 0 rc 0/0 00000100:00000001:0.0:1784948894.134342:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784948894.134350:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 728 00000100:00000200:0.0:1784948894.134351:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948894.134354:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948894.134358:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784948894.134374:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948894.134375:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948894.134384:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948894.134384:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000010:2.0:1784948894.134401:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009242fa2b. 00000100:00000200:0.0:1784948894.134408:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948894.134409:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 704 00000020:00000010:2.0:1784948894.134420:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000040:3.0:1784948894.134424:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948894.134436:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558294016 : -115544151257600 : ffff96e9c7e88e00) 00000020:00000010:2.0:1784948894.134438:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000100:00000001:0.0:1784948894.134443:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948894.134457:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784948894.134459:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000040:3.0:1784948894.134464:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009778919c x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:555/0 lens 224/0 e 0 to 0 dl 1784948905 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948894.134465:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948894.134472:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:0.0:1784948894.134474:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948894.134489:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000010:2.0:1784948894.134489:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c4a1e5eb. 00000100:00000001:3.0:1784948894.134492:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948894.134502:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948894.134505:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 02000000:00000001:2.0:1784948894.134506:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:0.0:1784948894.134510:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948894.134516:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948894.134519:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00001000:0.0:1784948894.134525:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=66 v=5 (5 5 5 5) 00000020:00000010:3.0:1784948894.134529:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000100:00000200:2.0:1784948894.134530:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:0.0:1784948894.134538:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=66 v=5 (1 1 1 1) 00000400:00000010:2.0:1784948894.134544:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000003793f8ab. 00000100:00000001:0.0:1784948894.134549:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00100000:3.0:1784948894.134557:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009778919c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289659904:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948894.134559:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948894.134561:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c55bc68f. 00010000:00000001:0.0:1784948894.134579:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:3.0:1784948894.134581:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289659904 00010000:00000001:0.0:1784948894.134583:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948894.134591:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948894.134592:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948894.134599:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:0.0:1784948894.134604:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948894.134606:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948894.134617:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948894.134622:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784948894.134632:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784948894.134633:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948894.134638:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948894.134640:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948894.134646:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948894.134648:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948894.134653:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:0.0:1784948894.134656:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948894.134658:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948894.134667:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948894.134675:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948894.134681:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:0.0:1784948894.134684:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289659776:0@lo:400: 02000000:00000010:3.0:1784948894.134687:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000aea1d93d. 02000000:00000001:3.0:1784948894.134695:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948894.134701:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948894.134704:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948894.134704:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948894.134715:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289659904 00000100:00000001:0.0:1784948894.134717:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:3.0:1784948894.134720:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:0.0:1784948894.134728:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b16a92e x1871651289659776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:3.0:1784948894.134730:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009778919c x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:555/0 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948894.134784:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948894.134789:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:3.0:1784948894.134790:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948894.134794:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00001000:3.0:1784948894.134796:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=61 v=5 (1 1 1 1) 02000000:00000010:0.0:1784948894.134799:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000b8810b40. 00000100:00000001:3.0:1784948894.134805:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784948894.134809:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784948894.134812:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000001:0.0:1784948894.134815:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784948894.134818:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:0.0:1784948894.134820:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 02000000:00000001:3.0:1784948894.134826:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:0.0:1784948894.134828:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:3.0:1784948894.134854:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948894.134862:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 00000100:00000001:3.0:1784948894.134862:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948894.134872:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784948894.134874:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:3.0:1784948894.134878:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 02000000:00000001:0.0:1784948894.134879:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948894.134884:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b16a92e. 00000100:00000200:3.0:1784948894.134890:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289659904, offset 224 00000100:00000001:0.0:1784948894.134891:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948894.134896:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948894.134938:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948894.134944:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948894.134953:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784948894.134955:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948894.134961:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948894.134968:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784948894.134969:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3354:3354:240:4294967295] 192.168.202.137@tcp LPNI seq info [2170:2170:8:4294967295] 00000100:00000001:0.0:1784948894.134973:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948894.134988:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948894.134997:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948894.135002:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948894.135008:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948894.135015:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948894.135018:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948894.135036:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948894.135040:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948894.135043:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948894.135045:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948894.135048:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948894.135053:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009778919c x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:555/0 lens 224/224 e 0 to 0 dl 1784948905 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948894.135064:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009778919c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289659904:12345-192.168.202.137@tcp:400: Request processed in 574us (1063us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948894.135074:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 704 00000100:00000040:3.0:1784948894.135078:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948894.135082:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948894.135084:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948894.135090:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000aa7a2a20. 00000020:00000010:3.0:1784948894.135096:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003c019a34. 00000020:00000010:3.0:1784948894.135101:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000898cc47a. 00000020:00000040:3.0:1784948894.135107:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948894.135111:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948894.135213:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948894.135219:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:1.0:1784948894.135225:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948894.135236:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948894.135237:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948894.135245:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000800:00000001:3.0:1784948894.135251:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948894.135253:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000800:00000001:3.0:1784948894.135257:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948894.135264:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784948894.135265:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.135273:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948894.135282:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948894.135300:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437a00 00000400:00000200:3.0:1784948894.135313:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x86c9 [1] + 224 00000800:00000001:3.0:1784948894.135323:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948894.135340:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948894.135344:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948894.135351:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948894.135360:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948894.135366:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948894.135385:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948894.135416:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948894.135428:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948894.135432:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.135483:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948894.135489:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948894.135496:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948894.135499:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948894.135503:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019017dd3. 00000400:00000200:1.0:1784948894.135510:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784948894.135514:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000100:00000001:1.0:1784948894.135519:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948894.135528:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948894.135546:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948894.135561:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948894.135564:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.135568:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948894.135571:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948894.135573:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948894.135576:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948894.135580:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948894.135587:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=61 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948894.135594:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=61 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948894.135600:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948894.135602:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948894.135606:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948894.135612:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948894.135627:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948894.135633:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948894.135635:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948894.135639:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948894.135641:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948894.135647:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948894.135661:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948894.135665:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948894.135668:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948894.135674:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948894.135689:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005dccb977 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289659904:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948894.135697:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.135701:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948894.135705:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005dccb977 x1871651289659904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948894.135713:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948894.135715:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948894.135717:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000043d62367. 02000000:00000001:1.0:1784948894.135721:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948894.135723:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948894.135725:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948894.135728:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948894.135729:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 02000000:00000001:1.0:1784948894.135733:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948894.135735:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948894.135736:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005dccb977. 00000100:00000001:1.0:1784948894.135740:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948894.135742:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.135744:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948894.135749:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948894.135751:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948894.135753:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948894.135755:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948894.176051:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305833515, at 4305833516 00000100:00000040:1.0:1784948894.176055:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305833515, at 4305833516 00000100:00000040:3.0:1784948894.176055:0:343821:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305833515, at 4305833516 00000100:00000010:0.0:1784948894.176069:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:3.0:1784948894.176073:0:343821:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000010:1.0:1784948894.176079:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:2.0:1784948894.176096:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305833515, at 4305833516 00000100:00000010:2.0:1784948894.176122:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00010000:00000001:2.0:1784948894.964119:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948894.964133:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948894.964144:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948894.964146:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948894.964293:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948894.964302:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948894.964308:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948895.156049:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948895.156065:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948895.156068:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948895.156075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948895.156079:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948895.156084:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948895.156088:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948895.988020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948895.988030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948895.988037:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948895.988039:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948895.988143:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948895.988155:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948895.988162:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948896.180018:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948896.180037:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948896.180041:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948896.180049:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948896.180053:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948896.180059:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948896.180064:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948897.012078:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948897.012090:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948897.012101:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948897.012103:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948897.012243:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948897.012252:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948897.012258:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948897.204035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948897.204054:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948897.204058:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948897.204066:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948897.204071:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948897.204077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948897.204082:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948898.036061:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948898.036073:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948898.036083:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948898.036085:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948898.036191:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948898.036200:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948898.036206:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948898.228068:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948898.228076:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784948898.228092:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948873/real 1784948873] req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948898 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784948898.228114:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948898.228117:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948898.228122:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d6b8e52f. 00000400:00000200:0.0:1784948898.228128:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:0.0:1784948898.228132:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000100:00000001:0.0:1784948898.228137:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948898.228141:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948898 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948898.228153:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948898 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948898.228163:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948898.228164:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.228168:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948898.228170:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948898.228174:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948898 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948898.228182:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.228192:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948898.228194:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.228197:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948898.228200:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948898.228203:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948898.228207:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948898 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948898.228215:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784948898.228219:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.228222:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948898.228224:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.228227:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784948898.228230:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784948898.228234:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784948898.228262:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:0.0:1784948898.228266:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784948898.228270:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948898 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948898.228282:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000e7e08a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289658752:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784948898.228289:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.228293:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948898.228298:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000e7e08a5 x1871651289658752/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948898 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948898.228307:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948898.228311:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948898.228316:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000002afa749d. 00000100:00000040:1.0:1784948898.228316:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1784948898.228327:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1784948898.228329:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000020:00000001:0.0:1784948898.228333:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948898.228337:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000010:1.0:1784948898.228339:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000069457997. 00000020:00000001:0.0:1784948898.228343:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000040:1.0:1784948898.228352:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 02000000:00000010:0.0:1784948898.228353:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000008d04782c. 00000100:00000001:1.0:1784948898.228365:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948898.228366:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948898.228373:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948898.228374:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948898.228381:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000e7e08a5. 02000000:00000001:1.0:1784948898.228384:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.228386:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948898.228394:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.228397:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784948898.228399:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000061dbde97. 00000100:00000001:0.0:1784948898.228402:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948898.228407:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.228411:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.228412:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948898.228417:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948898.228427:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948898.228442:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948898.228446:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948898.228451:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069457997] to pc [ptlrpcd_00_03+3] req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.228486:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948898.228490:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948898.228494:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948898.228497:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948898.228499:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948898.228504:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784948898.228509:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11145 00000400:00000200:1.0:1784948898.228519:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:1.0:1784948898.228524:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11144 00000400:00000200:1.0:1784948898.228530:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000040:1.0:1784948898.228536:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948898.228540:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948898.228546:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948898.228548:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948898.228550:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000001:1.0:1784948898.228559:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000100:00100000:0.0:1784948898.228561:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000020:00000040:1.0:1784948898.228562:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:1.0:1784948898.228573:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948898.228575:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948898.228575:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948898.228584:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:2.0:1784948898.228596:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948898.228597:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:0.0:1784948898.228598:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948898.228606:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948898.228619:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:0.0:1784948898.228620:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948898.228624:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948898.228630:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948898.228637:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000040:0.0:1784948898.228643:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.228649:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948898.228659:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:1.0:1784948898.228668:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:1.0:1784948898.228679:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:0.0:1784948898.228686:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1784948898.228691:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000001:0.0:1784948898.228695:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948898.228704:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 02000000:00000001:0.0:1784948898.228707:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948898.228712:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948898.228716:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948898.228720:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1784948898.228729:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948898.228735:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784948898.228736:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289660032:0@lo:400: 00000100:00000001:0.0:1784948898.228749:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948898.228749:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948898.228760:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948898.228760:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:0.0:1784948898.228765:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.228769:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948898.228771:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948898.228784:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000ead2c724. 00000100:00000010:1.0:1784948898.228784:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007bd58f9d. 02000000:00000001:0.0:1784948898.228795:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948898.228795:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000010:0.0:1784948898.228830:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ee18875c. 00000400:00000010:0.0:1784948898.228839:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784948898.228846:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289660032, portal 25 00000100:00000001:0.0:1784948898.228849:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948898.228851:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549155328 : -115544160396288 : ffff96e9c75d1c00) 00000100:00000040:0.0:1784948898.228859:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948898.228871:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1784948898.228879:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784948898.228880:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.228888:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1784948898.228889:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784948898.228893:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1784948898.228895:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 02000000:00000001:1.0:1784948898.228899:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948898.228902:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289660032, offset 0 02000000:00000010:1.0:1784948898.228943:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000fd4da236. 00000100:00000001:1.0:1784948898.228951:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948898.228961:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784948898.228979:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948898.228982:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948898.228989:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948898.228995:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bd58f9d] to pc [ptlrpcd_rcv+-1] req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948898.229007:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784948898.229033:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437a80 00000400:00000200:0.0:1784948898.229043:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7d59 [2] + 0 00000400:00000010:0.0:1784948898.229047:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000de368614. 00000100:00000001:1.0:1784948898.229054:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948898.229057:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000449b49f2 00000100:00000040:1.0:1784948898.229066:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1784948898.229074:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948898.229076:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:0.0:1784948898.229084:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000010:1.0:1784948898.229087:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000299dc118. 00000400:00000200:0.0:1784948898.229092:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000449b49f2 00000020:00000040:1.0:1784948898.229098:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000010:0.0:1784948898.229099:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000449b49f2. 00000100:00000001:0.0:1784948898.229110:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948898.229112:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1784948898.229115:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:2.0:1784948898.229118:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948898.229120:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1784948898.229123:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000774f7e3e x1871651289660032 msgsize 224 02000000:00000001:1.0:1784948898.229135:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948898.229136:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1784948898.229137:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:2.0:1784948898.229157:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1784948898.229158:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000040:0.0:1784948898.229159:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948898.229178:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948898.229194:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948898.229195:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948898.229197:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948898.229212:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784948898.229226:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784948898.229238:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948898.229253:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948898.229312:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948898.229323:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289660032 00000100:00000001:2.0:1784948898.229336:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1784948898.229336:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948898.229339:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948898.229341:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948898.229349:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948898.229350:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948898.229354:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948898.229359:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948898.229360:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948898.229371:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948898.229374:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948898.229386:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948898.229387:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000299dc118] to pc [ptlrpcd_00_00+0] req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948898.229388:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289660032 00000100:00000001:0.0:1784948898.229389:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948898.229403:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948898.229407:0:337121:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784948898.229412:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948898.229414:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948898.229419:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948898.229427:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948898.229436:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948898.229437:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305837611 00000100:00100000:2.0:1784948898.229442:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289660160:1.2.3.4@tcp:8: 00000020:00000001:3.0:1784948898.229450:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:0.0:1784948898.229458:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000020:00000001:3.0:1784948898.229462:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948898.229480:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948898.229483:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948898.229484:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000010:3.0:1784948898.229495:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:2.0:1784948898.229498:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.229503:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305837611 00000100:00000001:3.0:1784948898.229504:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948898.229510:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948898.229513:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000100:00000001:1.0:1784948898.229515:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948898.229520:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:2.0:1784948898.229532:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.229536:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1784948898.229537:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11175) 00000020:00000010:3.0:1784948898.229538:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000898cc47a. 00000100:00000001:2.0:1784948898.229542:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.229550:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305837611 00000020:00000010:3.0:1784948898.229555:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000aa7a2a20. 00000100:00000010:0.0:1784948898.229564:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:2.0:1784948898.229569:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:3.0:1784948898.229569:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003c019a34. 00000100:00000001:1.0:1784948898.229575:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948898.229578:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948898.229583:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.229588:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948898.229595:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948898.229597:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948898.229600:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305837611 00000100:00000001:1.0:1784948898.229600:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948898.229611:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948898.229611:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:0.0:1784948898.229616:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000040:1.0:1784948898.229625:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948898.229628:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.229630:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948898.229630:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948898.229639:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.229644:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948898.229649:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948898.229657:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.229669:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948898.229682:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.229683:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948898.229685:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.229692:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.229697:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948898.229704:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948898.229706:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948898.229713:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:1.0:1784948898.229714:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784948898.229723:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 729 00000100:00000001:0.0:1784948898.229726:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948898.229727:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948898.229731:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948898.229736:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948898.229738:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742033488 : -115538967518128 : ffff96eafce22450) 00000100:00100000:1.0:1784948898.229743:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289660288:192.168.202.137@tcp:400: 00000100:00000040:3.0:1784948898.229755:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000774f7e3e x1871651289660032/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:559/0 lens 224/0 e 0 to 0 dl 1784948909 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.229758:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948898.229769:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948898.229778:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:1.0:1784948898.229779:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948898.229786:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 02000000:00000001:1.0:1784948898.229789:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948898.229793:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948898.229828:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 02000000:00000010:1.0:1784948898.229830:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000043d62367. 00000100:00100000:3.0:1784948898.229838:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000774f7e3e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289660032:12345-0@lo:400: 02000000:00000001:1.0:1784948898.229842:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948898.229849:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289660032 00000020:00000001:3.0:1784948898.229855:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:1.0:1784948898.229857:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019017dd3. 00000020:00000001:3.0:1784948898.229861:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948898.229868:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948898.229871:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:1.0:1784948898.229873:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000020:00000001:3.0:1784948898.229874:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784948898.229885:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000200:1.0:1784948898.229887:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289660288, portal 10 00000020:00000001:3.0:1784948898.229888:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948898.229896:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948898.229899:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1784948898.229901:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948898.229943:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549152640 : -115544160398976 : ffff96e9c75d1180) 00000020:00000001:3.0:1784948898.229951:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948898.229958:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948898.229962:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:1.0:1784948898.229964:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948898.229969:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948898.229978:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000091dbaf3c. 02000000:00000001:3.0:1784948898.229986:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.229993:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948898.229993:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948898.230001:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948898.230008:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289660032 00000100:00000040:1.0:1784948898.230009:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00010000:00000001:3.0:1784948898.230014:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000010:1.0:1784948898.230020:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00010000:00000200:3.0:1784948898.230023:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000774f7e3e x1871651289660032/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:559/0 lens 224/224 e 0 to 0 dl 1784948909 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948898.230032:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289660288, offset 0 00010000:00000001:3.0:1784948898.230046:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:1.0:1784948898.230055:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00010000:00000001:3.0:1784948898.230055:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948898.230066:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=70 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948898.230073:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784948898.230078:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:3.0:1784948898.230083:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948898.230089:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:1.0:1784948898.230090:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1784948898.230097:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:1.0:1784948898.230103:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:3.0:1784948898.230104:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948898.230112:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948898.230113:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948898.230121:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:1.0:1784948898.230124:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000010:3.0:1784948898.230130:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000400:00000200:1.0:1784948898.230132:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3355:3355:240:4294967295] 192.168.202.137@tcp LPNI seq info [2171:2171:8:4294967295] 00000100:00000200:3.0:1784948898.230141:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289660032, offset 224 00000400:00000200:1.0:1784948898.230160:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000400:00000200:3.0:1784948898.230161:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000200:1.0:1784948898.230176:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948898.230184:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:1.0:1784948898.230187:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000400:00000200:3.0:1784948898.230203:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437a80 00000800:00000200:1.0:1784948898.230204:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948898.230221:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x86dd [1] + 224 00000800:00000200:1.0:1784948898.230229:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948898.230237:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1784948898.230238:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948898.230248:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948898.230253:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948898.230256:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.230262:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948898.230274:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2086 usecs (3 jiffies) 00000100:00000040:3.0:1784948898.230278:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.230281:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305837573 00000100:00000001:1.0:1784948898.230290:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305837611 00000100:00000001:1.0:1784948898.230299:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305837573 00000100:00000001:1.0:1784948898.230308:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305837611 00000100:00000001:1.0:1784948898.230311:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305837573 00000100:00000001:1.0:1784948898.230314:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305837611 00000100:00000001:1.0:1784948898.230317:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305837573 00000100:00000001:1.0:1784948898.230320:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305837611 00000100:00000001:1.0:1784948898.230322:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948898.230326:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948898.230330:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948898.230332:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.230337:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948898.230342:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000100:00000001:1.0:1784948898.230345:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:3.0:1784948898.230347:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000100:00000001:1.0:1784948898.230351:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948898.230355:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948898.230357:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948898.230361:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948898.230363:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948898.230371:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948898.230373:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.230375:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948898.230385:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:3.0:1784948898.230390:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948898.230393:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948898.230397:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948898.230403:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774f7e3e x1871651289660032/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:559/0 lens 224/224 e 0 to 0 dl 1784948909 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948898.230421:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000774f7e3e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289660032:12345-0@lo:400: Request processed in 637us (1290us total) trans 0 rc 0/0 00000100:00000001:2.0:1784948898.230427:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784948898.230436:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 729 00000100:00000001:2.0:1784948898.230437:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948898.230441:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948898.230447:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948898.230449:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948898.230455:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000aa7a2a20. 00000400:00000001:2.0:1784948898.230460:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:3.0:1784948898.230462:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003c019a34. 00000800:00000200:0.0:1784948898.230469:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000001:2.0:1784948898.230470:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:3.0:1784948898.230472:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000898cc47a. 00000800:00000010:0.0:1784948898.230483:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000400:00000010:2.0:1784948898.230486:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ee18875c. 00000020:00000040:3.0:1784948898.230486:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:0.0:1784948898.230500:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784948898.230504:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000800:00000001:1.0:1784948898.230505:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948898.230510:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 02000000:00000010:3.0:1784948898.230512:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000091dbaf3c. 00000400:00000010:2.0:1784948898.230521:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000400:00000200:0.0:1784948898.230523:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784948898.230524:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000800:00000001:1.0:1784948898.230525:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948898.230535:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948898.230541:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784948898.230548:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948898.230549:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1784948898.230551:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000800:00000001:1.0:1784948898.230551:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948898.230566:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000400:00000010:3.0:1784948898.230567:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a51bb92. 00000100:00000200:2.0:1784948898.230575:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948898.230582:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948898.230587:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:3.0:1784948898.230594:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008c74c22d. 00000100:00000200:0.0:1784948898.230609:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948898.230619:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437b80 00000400:00000200:1.0:1784948898.230657:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 158960 00000100:00000200:2.0:1784948898.230679:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948898.230683:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948898.230699:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.230709:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948898.230716:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784948898.230733:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948898.230748:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948898.230758:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948898.230766:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784948898.230770:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948898.230774:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784948898.230775:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948898.230783:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784948898.230786:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:1.0:1784948898.230786:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000926235a7. 00000100:00000040:1.0:1784948898.230821:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000926235a7 x1871651289660288 msgsize 224 02000000:00000001:2.0:1784948898.230829:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948898.230840:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:1.0:1784948898.230841:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784948898.230852:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948898.230863:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.230866:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00001000:2.0:1784948898.230876:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=70 v=5 (5 5 5 5) 00000800:00000001:1.0:1784948898.230879:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948898.230887:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784948898.230894:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=70 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948898.230942:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948898.230948:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289660288 02000000:00000001:0.0:1784948898.230952:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948898.230956:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948898.230959:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948898.230965:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948898.230970:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00100000:0.0:1784948898.230971:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289660288 00000020:00000001:0.0:1784948898.230983:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784948898.230984:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948898.230986:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948898.230998:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948898.231003:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000001:2.0:1784948898.231003:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:0.0:1784948898.231014:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:2.0:1784948898.231016:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948898.231028:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948898.231033:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.231035:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948898.231044:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:0.0:1784948898.231050:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948898.231057:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948898.231058:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000020:00000010:0.0:1784948898.231072:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f11303f8. 00000020:00000010:0.0:1784948898.231086:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f57325dd. 00000020:00000010:0.0:1784948898.231095:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948898.231112:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948898.231124:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948898.231133:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948898.231140:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948898.231150:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948898.231154:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948898.231160:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948898.231162:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948898.231177:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948898.231179:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948898.231200:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948898.231209:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948898.231232:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948898.231238:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948898.231240:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289660032:0@lo:400: 00000100:00100000:0.0:1784948898.231262:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 705 00000100:00000001:1.0:1784948898.231264:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948898.231267:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.231271:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784948898.231274:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948898.231282:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.231287:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549158912 : -115544160392704 : ffff96e9c75d2a00) 00000100:00000001:2.0:1784948898.231288:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948898.231290:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948898.231312:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000926235a7 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:559/0 lens 224/0 e 0 to 0 dl 1784948909 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948898.231317:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000069457997 x1871651289660032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948898.231373:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948898.231384:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:2.0:1784948898.231389:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784948898.231391:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948898.231399:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 02000000:00000001:2.0:1784948898.231399:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948898.231413:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000ead2c724. 00000100:00100000:0.0:1784948898.231423:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000926235a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289660288:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948898.231433:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289660288 02000000:00000001:2.0:1784948898.231436:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948898.231437:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948898.231451:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:0.0:1784948898.231452:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948898.231457:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948898.231463:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948898.231464:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948898.231467:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948898.231477:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948898.231480:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:0.0:1784948898.231482:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784948898.231488:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000061dbde97. 00000020:00000001:0.0:1784948898.231493:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948898.231500:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:2.0:1784948898.231505:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1784948898.231506:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948898.231513:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1784948898.231516:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000010:2.0:1784948898.231521:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000069457997. 00000100:00000001:0.0:1784948898.231527:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948898.231534:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1784948898.231535:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948898.231541:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784948898.231548:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000af0b2fc8. 00000100:00000001:2.0:1784948898.231549:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948898.231563:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948898.231569:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948898.231574:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948898.231576:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948898.231596:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948898.231598:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000040:0.0:1784948898.231603:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289660288 00000100:00000001:2.0:1784948898.231607:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948898.231609:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948898.231618:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000926235a7 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:559/0 lens 224/224 e 0 to 0 dl 1784948909 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948898.231632:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948898.231637:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948898.231642:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=65 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948898.231651:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948898.231655:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948898.231660:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948898.231664:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948898.231667:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948898.231671:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948898.231677:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948898.231683:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000200:0.0:1784948898.231689:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289660288, offset 224 00000400:00000200:0.0:1784948898.231698:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948898.231712:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948898.231721:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3356:3356:240:4294967295] 192.168.202.137@tcp LPNI seq info [2172:2172:8:4294967295] 00000400:00000200:0.0:1784948898.231738:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948898.231745:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948898.231750:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d1b12746. 00000800:00000200:0.0:1784948898.231756:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948898.231762:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948898.231766:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d1b12746 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948898.231787:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948898.231791:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948898.231794:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948898.231817:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948898.231820:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948898.231826:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000926235a7 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:559/0 lens 224/224 e 0 to 0 dl 1784948909 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948898.231839:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000926235a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289660288:12345-192.168.202.137@tcp:400: Request processed in 465us (1017us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948898.231849:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 705 00000100:00000040:0.0:1784948898.231852:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948898.231855:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948898.231857:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948898.231861:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f57325dd. 00000020:00000010:0.0:1784948898.231865:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948898.231869:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000f11303f8. 00000020:00000040:0.0:1784948898.231874:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948898.231878:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948898.232047:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948898.232055:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d1b12746. 00000400:00000200:3.0:1784948898.232062:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948898.232071:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948898.232073:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948898.232078:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:3.0:1784948898.232083:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000800:00000001:0.0:1784948898.232087:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948898.232088:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948898.232093:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948898.232096:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948898.232104:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948898.232112:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948898.232125:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437b80 00000400:00000200:0.0:1784948898.232133:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x86e5 [1] + 224 00000800:00000001:0.0:1784948898.232139:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948898.232148:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948898.232151:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948898.232155:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948898.232161:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948898.232165:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948898.232177:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948898.232198:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948898.232205:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948898.232208:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.232309:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948898.232316:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948898.232323:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948898.232326:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948898.232329:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019017dd3. 00000400:00000200:1.0:1784948898.232336:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784948898.232338:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000100:00000001:1.0:1784948898.232342:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948898.232349:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948898.232363:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.232373:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948898.232375:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.232378:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948898.232381:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948898.232384:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948898.232386:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948898.232389:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948898.232395:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=65 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948898.232400:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=65 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948898.232403:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948898.232405:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948898.232408:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948898.232413:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948898.232422:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.232426:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948898.232428:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948898.232430:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948898.232432:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948898.232437:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.232445:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948898.232448:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948898.232450:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948898.232453:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948898.232462:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289660288:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948898.232467:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.232469:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948898.232472:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000299dc118 x1871651289660288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948914 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948898.232480:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948898.232482:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948898.232484:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000043d62367. 02000000:00000001:1.0:1784948898.232489:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948898.232491:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948898.232493:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948898.232496:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948898.232498:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 02000000:00000001:1.0:1784948898.232502:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948898.232504:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948898.232506:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000299dc118. 00000100:00000001:1.0:1784948898.232509:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948898.232511:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.232513:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948898.232519:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948898.232521:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948898.232523:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948898.232525:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948898.271977:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305837611, at 4305837612 00000100:00000010:0.0:1784948898.271994:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:3.0:1784948898.272060:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305837611, at 4305837612 00000100:00000010:3.0:1784948898.272074:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:1.0:1784948898.272088:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305837611, at 4305837612 00000100:00000010:1.0:1784948898.272109:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000040:2.0:1784948898.272136:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305837611, at 4305837612 00000100:00000010:2.0:1784948898.272158:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00010000:00000001:2.0:1784948899.060018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948899.060028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948899.060042:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948899.060045:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948899.060159:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948899.060168:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948899.060174:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948899.252060:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948899.252071:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784948899.252095:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948898/real 0] req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948899 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948899.252124:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.252130:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948899.252134:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948899.252140:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948899 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948899.252152:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.252167:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948899.252169:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948899.252173:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948899.252177:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948899.252180:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948899.252184:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948899 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948899.252194:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784948899.252199:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.252201:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948899.252202:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.252205:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784948899.252208:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784948899.252213:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784948899.252248:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:2.0:1784948899.252253:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784948899.252257:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948899 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948899.252266:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289660160:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784948899.252277:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.252282:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948899.252285:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007bd58f9d x1871651289660160/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948899 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948899.252295:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1784948899.252299:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948899.252302:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784948899.252306:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948899.252312:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000fd4da236. 02000000:00000001:2.0:1784948899.252323:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948899.252328:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948899.252330:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007bd58f9d. 00000100:00000001:2.0:1784948899.252337:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948899.252339:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.252343:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.252348:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948899.252350:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948899.252353:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948899.252356:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.252377:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948899.252390:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948899.252401:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000299dc118. 00000020:00000040:1.0:1784948899.252409:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948899.252415:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948899.252419:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948899.252427:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948899.252436:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000001:1.0:1784948899.252445:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.252456:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948899.252471:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948899.252474:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.252480:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000299dc118] to pc [ptlrpcd_00_01+1] req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948899.252522:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948899.252526:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948899.252530:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784948899.252533:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784948899.252537:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784948899.252542:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784948899.252548:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11145 00000400:00000200:1.0:1784948899.252560:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:1.0:1784948899.252566:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11170 00000400:00000200:1.0:1784948899.252572:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:1.0:1784948899.252579:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1784948899.252580:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948899.252588:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00100000:2.0:1784948899.252590:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:1.0:1784948899.252595:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948899.252597:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948899.252607:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948899.252610:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948899.252613:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784948899.252615:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948899.252621:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948899.252634:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784948899.252635:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948899.252638:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948899.252650:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:2.0:1784948899.252650:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948899.252658:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1784948899.252659:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948899.252670:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000040:2.0:1784948899.252676:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948899.252685:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:1.0:1784948899.252698:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:1.0:1784948899.252722:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:2.0:1784948899.252730:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948899.252731:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948899.252742:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.252749:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948899.252754:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948899.252756:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948899.252770:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784948899.252771:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:1.0:1784948899.252821:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 02000000:00000001:2.0:1784948899.252824:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948899.252830:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00100000:2.0:1784948899.252837:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289660416:0@lo:400: 00000100:00000001:1.0:1784948899.252838:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948899.252848:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1784948899.252859:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948899.252860:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948899.252870:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948899.252872:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948899.252873:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1784948899.252883:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.252884:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948899.252950:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ddd830de. 02000000:00000001:2.0:1784948899.252956:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1784948899.252959:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:1.0:1784948899.252964:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1784948899.252966:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948899.252969:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000229221cc. 00000100:00000001:1.0:1784948899.252971:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948899.252987:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948899.252989:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948899.253003:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948899.253009:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000ef44eaa. 02000000:00000010:1.0:1784948899.253012:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000009265644e. 00000100:00000001:1.0:1784948899.253024:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948899.253029:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00100000:1.0:1784948899.253039:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1784948899.253048:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289660416, portal 25 00000100:00000001:1.0:1784948899.253071:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948899.253072:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948899.253087:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549152640 : -115544160398976 : ffff96e9c75d1180) 00000100:00000001:1.0:1784948899.253089:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.253112:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ddd830de] to pc [ptlrpcd_rcv+-1] req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948899.253114:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948899.253237:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948899.253247:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948899.253250:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000100:00000200:2.0:1784948899.253255:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289660416, offset 0 00000100:00000001:1.0:1784948899.253256:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.253268:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948899.253272:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:2.0:1784948899.253278:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000010:1.0:1784948899.253283:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000002cc70ea. 00000020:00000040:1.0:1784948899.253290:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948899.253294:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948899.253296:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948899.253299:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948899.253309:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e79c6895. 00000400:00000200:2.0:1784948899.253309:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784948899.253323:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.253331:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784948899.253340:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437c00 00000100:00000001:0.0:1784948899.253367:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948899.253369:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1784948899.253371:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7d79 [2] + 0 00000100:00000001:0.0:1784948899.253378:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948899.253384:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.253392:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:2.0:1784948899.253396:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d5071549. 00000100:00000040:1.0:1784948899.253404:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000002cc70ea] to pc [ptlrpcd_00_02+2] req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948899.253411:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948899.253413:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c825fe8d 00000400:00000200:2.0:1784948899.253439:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948899.253441:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948899.253453:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.253454:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948899.253467:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784948899.253471:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948899.253478:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948899.253479:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11176) 00000400:00000200:2.0:1784948899.253481:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c825fe8d 02000000:00000001:0.0:1784948899.253484:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948899.253494:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000c825fe8d. 00000100:00100000:0.0:1784948899.253503:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289660544:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784948899.253505:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948899.253514:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784948899.253515:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784948899.253517:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1784948899.253532:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000040:2.0:1784948899.253533:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000b4879340 x1871651289660416 msgsize 224 02000000:00000001:0.0:1784948899.253535:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948899.253543:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948899.253555:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948899.253556:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948899.253558:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948899.253572:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948899.253577:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d02c7fef. 00000100:00000001:1.0:1784948899.253581:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1784948899.253586:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:1.0:1784948899.253591:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948899.253593:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948899.253599:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948899.253602:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948899.253607:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:0.0:1784948899.253608:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000047084e78. 00000100:00000001:1.0:1784948899.253612:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948899.253621:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:0.0:1784948899.253623:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00000040:1.0:1784948899.253630:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948899.253639:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000100:00000200:0.0:1784948899.253641:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289660544, portal 4 00000100:00000001:1.0:1784948899.253648:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:2.0:1784948899.253650:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000100:00000001:0.0:1784948899.253654:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948899.253658:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948899.253661:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.253668:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549156224 : -115544160395392 : ffff96e9c75d1f80) 00000100:00000001:2.0:1784948899.253670:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1784948899.253670:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289660416 02000000:00000001:1.0:1784948899.253672:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948899.253686:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948899.253686:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000200:2.0:1784948899.253692:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948899.253695:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:1.0:1784948899.253701:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948899.253702:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948899.253704:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948899.253725:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000002cc70ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289660672:192.168.202.137@tcp:400: 02000000:00000001:3.0:1784948899.253727:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948899.253744:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289660416 00000020:00000001:3.0:1784948899.253758:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948899.253766:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1784948899.253770:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 02000000:00000001:1.0:1784948899.253779:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1784948899.253785:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1784948899.253811:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948899.253820:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 02000000:00000001:1.0:1784948899.253827:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948899.253832:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1784948899.253834:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000010:1.0:1784948899.253837:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000043d62367. 00000100:00000001:0.0:1784948899.253840:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948899.253850:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948899.253852:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948899.253854:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:0.0:1784948899.253865:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948899.253871:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019017dd3. 00000100:00000001:3.0:1784948899.253876:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:0.0:1784948899.253882:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 00000400:00000010:1.0:1784948899.253893:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000020:00000010:3.0:1784948899.253894:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b4985bb9. 00000100:00000200:0.0:1784948899.253902:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289660544, offset 0 00000100:00000001:3.0:1784948899.253947:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948899.253952:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b4985bb9. 00000020:00000010:3.0:1784948899.253965:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000898cc47a. 00000400:00000200:0.0:1784948899.253985:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000010:3.0:1784948899.253987:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000aa7a2a20. 00000100:00000001:2.0:1784948899.253989:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948899.254000:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:1.0:1784948899.254003:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289660672, portal 10 00000020:00000010:3.0:1784948899.254004:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003c019a34. 00000400:00000200:0.0:1784948899.254015:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784948899.254016:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1784948899.254022:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948899.254023:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549163392 : -115544160388224 : ffff96e9c75d3b80) 00000400:00000200:0.0:1784948899.254029:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948899.254033:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948899.254037:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784948899.254042:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:1.0:1784948899.254044:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948899.254045:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948899.254054:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948899.254064:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948899.254066:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784948899.254081:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784948899.254082:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3357:3357:240:4294967295] 192.168.202.137@tcp LPNI seq info [2173:2173:8:4294967295] 00000100:00000001:3.0:1784948899.254090:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948899.254094:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948899.254101:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000200:1.0:1784948899.254113:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289660672, offset 0 00000100:00000001:3.0:1784948899.254114:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784948899.254115:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948899.254123:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784948899.254132:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948899.254139:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948899.254140:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 730 00000800:00000200:0.0:1784948899.254146:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948899.254157:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:3.0:1784948899.254158:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948899.254160:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305838636 00000800:00000010:0.0:1784948899.254164:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000086e4d21. 00000400:00000200:1.0:1784948899.254170:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948899.254186:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742030416 : -115538967521200 : ffff96eafce21850) 00000400:00000200:1.0:1784948899.254193:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000010:2.0:1784948899.254193:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008cd34e64. 00000800:00000200:0.0:1784948899.254197:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:3.0:1784948899.254206:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b4879340 x1871651289660416/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:560/0 lens 224/0 e 0 to 0 dl 1784948910 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948899.254219:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784948899.254222:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000800:00000200:0.0:1784948899.254241:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:1.0:1784948899.254243:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:2.0:1784948899.254249:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305838636 00000800:00000200:0.0:1784948899.254256:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000086e4d21 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1784948899.254262:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784948899.254264:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3358:3358:239:4294967295] 192.168.202.137@tcp LPNI seq info [2174:2174:7:4294967295] 00000100:00000010:2.0:1784948899.254266:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f99e6ef2. 00000020:00000010:3.0:1784948899.254272:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b4985bb9. 00000100:00000001:2.0:1784948899.254288:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948899.254291:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948899.254292:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948899.254310:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305838636 00000020:00000010:3.0:1784948899.254312:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b4985bb9. 00000100:00000040:0.0:1784948899.254314:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000400:00000200:1.0:1784948899.254333:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948899.254340:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305838636 00000100:00000010:2.0:1784948899.254342:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003b6d02be. 00000100:00100000:3.0:1784948899.254344:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b4879340 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289660416:12345-0@lo:400: 00000800:00000200:1.0:1784948899.254358:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948899.254369:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305838636 00000100:00000001:2.0:1784948899.254372:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000200:3.0:1784948899.254374:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289660416 00000800:00000010:1.0:1784948899.254375:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c2b2d4f6. 00000100:00000001:0.0:1784948899.254378:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305838636 00000020:00000001:3.0:1784948899.254392:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948899.254398:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305838636 00000020:00000001:3.0:1784948899.254403:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784948899.254406:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305838636 00000100:00000001:2.0:1784948899.254408:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305838636 00000800:00000200:1.0:1784948899.254414:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:3.0:1784948899.254414:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.254427:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305838636 00000100:00000010:2.0:1784948899.254427:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 00000020:00000001:3.0:1784948899.254431:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948899.254445:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:1.0:1784948899.254447:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:0.0:1784948899.254451:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305838636 00000100:00000001:2.0:1784948899.254451:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000001:3.0:1784948899.254457:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000200:1.0:1784948899.254463:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c2b2d4f6 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:3.0:1784948899.254468:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948899.254470:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305838636 00000100:00000001:2.0:1784948899.254470:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948899.254487:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948899.254488:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948899.254494:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.254496:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948899.254497:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:1.0:1784948899.254501:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2087 usecs (3 jiffies) 00000020:00000001:3.0:1784948899.254512:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948899.254516:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.254518:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305838597 00000100:00000001:0.0:1784948899.254519:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948899.254524:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948899.254532:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.254536:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305838636 00000100:00000001:0.0:1784948899.254540:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948899.254541:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948899.254543:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305838597 00000100:00000001:0.0:1784948899.254552:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.254553:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305838636 02000000:00000001:3.0:1784948899.254553:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948899.254565:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305838597 02000000:00000010:3.0:1784948899.254565:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d8bbc1a8. 00000100:00000001:0.0:1784948899.254568:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784948899.254576:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.254579:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1784948899.254580:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305838636 00000100:00000001:3.0:1784948899.254582:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.254593:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305838597 00000100:00000001:0.0:1784948899.254594:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948899.254595:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.254602:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305838636 00000100:00000001:0.0:1784948899.254603:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000040:3.0:1784948899.254604:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289660416 00000100:00000001:0.0:1784948899.254613:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.254614:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784948899.254615:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948899.254624:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784948899.254627:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b4879340 x1871651289660416/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:560/0 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948899.254628:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.254639:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.254650:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948899.254652:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948899.254658:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948899.254662:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=71 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948899.254668:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948899.254671:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948899.254674:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948899.254678:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948899.254680:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948899.254685:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948899.254689:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948899.254695:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000100:00000200:3.0:1784948899.254701:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289660416, offset 224 00000400:00000200:3.0:1784948899.254708:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784948899.254720:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:2.0:1784948899.254743:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1784948899.254743:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437c00 00000800:00000010:2.0:1784948899.254757:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000086e4d21. 00000400:00000200:3.0:1784948899.254769:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x86f9 [1] + 224 00000400:00000200:2.0:1784948899.254772:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948899.254817:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948899.254820:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948899.254825:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000400:00000010:2.0:1784948899.254835:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000400:00000200:3.0:1784948899.254836:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:1.0:1784948899.254848:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948899.254854:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948899.254862:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948899.254866:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948899.254876:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948899.254975:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948899.254991:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c2b2d4f6. 00000400:00000200:0.0:1784948899.255009:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948899.255013:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784948899.255025:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948899.255031:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948899.255040:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948899.255042:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948899.255044:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948924 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948899.255045:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948899.255065:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:0.0:1784948899.255077:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000400:00000200:1.0:1784948899.255083:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948899.255094:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948899.255115:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948899.255138:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6437c80 00000100:00000001:2.0:1784948899.255176:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1784948899.255178:0:337114:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289660544 offset 0 length 520: 4 00000100:00000001:2.0:1784948899.255193:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948899.255196:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948899.255198:0:337114:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000ede77a6c is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:1.0:1784948899.255209:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:2.0:1784948899.255215:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948899.255216:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948899.255229:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948899.255230:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000800:00000001:2.0:1784948899.255235:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948899.255237:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784948899.255244:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000800:00000001:2.0:1784948899.255253:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948899.255260:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948899.255267:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948899.255268:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784948899.255275:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948899.255276:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948899.255279:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784948899.255287:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948899.255292:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784948899.255301:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948899.255303:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948899.255322:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:3.0:1784948899.255336:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:2.0:1784948899.255338:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437d00 00000100:00000001:1.0:1784948899.255340:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948899.255353:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.255355:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784948899.255368:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 159184 00000020:00000001:3.0:1784948899.255369:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948899.255386:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b4879340 x1871651289660416/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:560/0 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948899.255387:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:2.0:1784948899.255387:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:1.0:1784948899.255402:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948899.255408:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000ef44eaa. 00000800:00000001:2.0:1784948899.255415:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948899.255423:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948899.255425:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000100:00100000:3.0:1784948899.255425:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b4879340 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289660416:12345-0@lo:400: Request processed in 1159us (1888us total) trans 0 rc 0/0 00000400:00000010:1.0:1784948899.255446:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00100000:3.0:1784948899.255463:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 730 00000100:00000001:1.0:1784948899.255465:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1784948899.255465:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:1.0:1784948899.255487:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948899.255491:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784948899.255494:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948899.255506:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784948899.255508:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948899.255519:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:2.0:1784948899.255525:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000edab6e1. 00000020:00000010:3.0:1784948899.255529:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000aa7a2a20. 00000100:00000200:1.0:1784948899.255537:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948899.255538:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000000edab6e1 x1871651289660672 msgsize 224 00000020:00000010:3.0:1784948899.255542:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003c019a34. 00000020:00000010:3.0:1784948899.255561:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000898cc47a. 00000020:00000040:3.0:1784948899.255579:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00100000:2.0:1784948899.255588:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:3.0:1784948899.255593:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784948899.255607:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d8bbc1a8. 00000100:00000001:1.0:1784948899.255615:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1784948899.255616:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784948899.255624:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948899.255625:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948899.255629:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948899.255645:0:337119:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:3.0:1784948899.255648:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000800:00000001:2.0:1784948899.255652:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784948899.255654:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:3.0:1784948899.255658:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d97ec0aa. 00000100:00000001:1.0:1784948899.255669:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948899.255675:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948899.255675:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948899.255677:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000b56e9af. 02000000:00000001:1.0:1784948899.255684:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948899.255706:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=71 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948899.255716:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=71 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948899.255720:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948899.255722:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948899.255724:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948899.255732:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00100000:0.0:1784948899.255734:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289660672 00010000:00000001:1.0:1784948899.255736:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948899.255746:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948899.255746:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.255753:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:1.0:1784948899.255759:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948899.255761:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948899.255772:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948899.255781:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289660672 00000100:00000001:1.0:1784948899.255783:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784948899.255787:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948899.255815:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1784948899.255816:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:1.0:1784948899.255821:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948899.255823:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948899.255831:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:1.0:1784948899.255833:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948899.255840:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948899.255850:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948899.255859:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:1.0:1784948899.255867:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289660416:0@lo:400: 00000020:00000010:0.0:1784948899.255869:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:0.0:1784948899.255881:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948899.255882:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948899.255885:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:1.0:1784948899.255890:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1784948899.255896:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f11303f8. 00000100:00000040:1.0:1784948899.255896:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000299dc118 x1871651289660416/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948899.255944:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f57325dd. 00000020:00000010:0.0:1784948899.255951:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948899.255960:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948899.255965:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784948899.255967:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784948899.255971:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1784948899.255973:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000010:1.0:1784948899.255978:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000229221cc. 00000100:00000001:0.0:1784948899.255980:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948899.255989:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784948899.255992:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000020:00000001:1.0:1784948899.255994:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:0.0:1784948899.256000:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000040:1.0:1784948899.256001:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948899.256008:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948899.256010:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948899.256017:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 02000000:00000001:1.0:1784948899.256021:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948899.256023:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948899.256025:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000299dc118. 00000100:00000001:1.0:1784948899.256030:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948899.256031:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256033:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256040:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256047:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948899.256051:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948899.256052:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.256059:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948899.256061:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948899.256067:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948899.256074:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 706 00000100:00000040:0.0:1784948899.256079:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948899.256083:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470025984 : -115544239525632 : ffff96e9c2a5b100) 00000100:00000040:0.0:1784948899.256091:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000edab6e1 x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:560/0 lens 224/0 e 0 to 0 dl 1784948910 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948899.256104:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948899.256108:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:0.0:1784948899.256113:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948899.256116:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00100000:0.0:1784948899.256122:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000edab6e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289660672:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948899.256126:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289660672 00000020:00000001:0.0:1784948899.256129:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948899.256133:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948899.256136:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948899.256138:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948899.256140:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948899.256144:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948899.256147:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948899.256149:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948899.256151:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948899.256153:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948899.256156:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948899.256160:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948899.256162:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948899.256166:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000004d27d87b. 02000000:00000001:0.0:1784948899.256168:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.256171:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948899.256173:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948899.256176:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289660672 00010000:00000001:0.0:1784948899.256179:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948899.256183:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000edab6e1 x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:560/0 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948899.256192:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948899.256195:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948899.256197:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=66 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948899.256202:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948899.256205:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948899.256207:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948899.256210:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948899.256212:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948899.256215:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948899.256218:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948899.256222:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f43fd4d9. 00000100:00000200:0.0:1784948899.256226:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289660672, offset 224 00000400:00000200:0.0:1784948899.256231:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948899.256242:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948899.256249:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3359:3359:240:4294967295] 192.168.202.137@tcp LPNI seq info [2175:2175:8:4294967295] 00000400:00000200:0.0:1784948899.256261:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948899.256267:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948899.256271:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006487a147. 00000800:00000200:0.0:1784948899.256276:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948899.256283:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948899.256287:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000006487a147 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948899.256304:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948899.256308:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948899.256311:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948899.256313:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948899.256316:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948899.256319:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000edab6e1 x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:560/0 lens 224/224 e 0 to 0 dl 1784948910 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948899.256332:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000edab6e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289660672:12345-192.168.202.137@tcp:400: Request processed in 225us (792us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948899.256345:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 706 00000100:00000040:0.0:1784948899.256349:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948899.256352:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948899.256355:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948899.256359:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f57325dd. 00000020:00000010:0.0:1784948899.256364:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948899.256368:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000f11303f8. 00000020:00000040:0.0:1784948899.256372:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948899.256376:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948899.256432:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948899.256439:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006487a147. 00000400:00000200:1.0:1784948899.256447:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948899.256448:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948899.256456:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948899.256463:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948899.256464:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f43fd4d9 00000400:00000010:1.0:1784948899.256470:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f43fd4d9. 00000800:00000001:0.0:1784948899.256473:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948899.256477:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784948899.256480:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256483:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784948899.256495:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948899.256509:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437d00 00000400:00000200:0.0:1784948899.256517:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8709 [1] + 224 00000800:00000001:0.0:1784948899.256524:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948899.256530:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948899.256533:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948899.256537:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948899.256543:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948899.256547:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948899.256559:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948899.256580:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948899.256588:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948899.256591:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256631:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948899.256636:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948899.256640:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948899.256643:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948899.256645:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019017dd3. 00000400:00000200:1.0:1784948899.256650:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784948899.256652:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000100:00000001:1.0:1784948899.256655:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948899.256661:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948899.256671:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948899.256680:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948899.256682:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256684:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948899.256686:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948899.256688:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948899.256689:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948899.256692:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948899.256698:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=66 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948899.256701:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=66 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948899.256704:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948899.256706:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948899.256708:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948899.256712:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948899.256721:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.256724:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948899.256726:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948899.256728:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948899.256729:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948899.256733:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948899.256741:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948899.256743:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948899.256745:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948899.256749:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948899.256757:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000002cc70ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289660672:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948899.256761:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256764:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948899.256767:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000002cc70ea x1871651289660672/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948899.256775:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948899.256777:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948899.256778:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000043d62367. 02000000:00000001:1.0:1784948899.256782:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948899.256783:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948899.256785:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948899.256813:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948899.256815:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e79c6895. 02000000:00000001:1.0:1784948899.256819:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948899.256821:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948899.256822:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000002cc70ea. 00000100:00000001:1.0:1784948899.256825:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948899.256827:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256829:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948899.256834:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948899.256835:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948899.256838:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948899.256840:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948899.297008:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305838636, at 4305838637 00000100:00000010:2.0:1784948899.297020:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003b6d02be. 00000100:00000040:0.0:1784948899.297026:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305838636, at 4305838637 00000100:00000040:3.0:1784948899.297036:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305838636, at 4305838637 00000100:00000010:0.0:1784948899.297046:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00000100:00000010:3.0:1784948899.297053:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00000100:00000040:1.0:1784948899.297111:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305838636, at 4305838637 00000100:00000010:1.0:1784948899.297124:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f99e6ef2. 00010000:00000001:2.0:1784948900.084033:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948900.084045:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948900.084053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948900.084057:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948900.084062:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948900.084065:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948900.084069:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948900.084072:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948900.084074:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948900.084076:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948900.084206:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948900.084217:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948900.084225:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948901.108095:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948901.108108:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948901.108118:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948901.108120:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948901.108239:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948901.108250:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948901.108258:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948902.132046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948902.132058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948902.132070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948902.132073:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948902.132173:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948902.132180:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948902.132185:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1784948902.516061:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000800:00000010:3.0:1784948902.516068:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e6a3ade8. 00000400:00000200:3.0:1784948902.516078:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948902.516087:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948902.516099:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11175 health :900 00000400:00000200:3.0:1784948902.516109:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948902.516118:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:3.0:1784948902.516126:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000800:00000100:3.0:1784948902.516164:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000800:00000010:3.0:1784948902.516169:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d625f2e8. 00000400:00000200:3.0:1784948902.516175:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948902.516183:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948902.516188:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948902.516193:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:3.0:1784948902.516199:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:0.0:1784948902.516255:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948902.516267:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:0.0:1784948902.516275:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948902.516278:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948902.516282:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:0.0:1784948902.516285:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000400:00000200:0.0:1784948902.516291:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948902.516295:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:0.0:1784948902.516301:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c49404b8. 00000400:00000200:0.0:1784948902.516307:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:0.0:1784948902.516310:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:0.0:1784948902.516316:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000400:00000200:0.0:1784948902.516320:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:0.0:1784948902.516323:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948902.516325:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948902.516327:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:0.0:1784948902.516328:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000400:00000200:0.0:1784948902.516331:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948902.516333:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:0.0:1784948902.516336:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004b465f54. 00000400:00000200:0.0:1784948902.516342:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:0.0:1784948902.516344:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:0.0:1784948902.516347:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00010000:00000001:2.0:1784948903.155972:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948903.155979:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948903.155984:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948903.155985:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948903.156073:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948903.156077:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948903.156080:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:1.0:1784948903.539973:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948903.539982:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000090a03b90. 00000400:00000010:1.0:1784948903.539991:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000043d62367. 00000400:00000010:1.0:1784948903.540011:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000400:00000010:1.0:1784948903.540021:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000991db8b9. 00000400:00000200:1.0:1784948903.540029:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000800:00000100:3.0:1784948903.540032:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000800:00000010:3.0:1784948903.540038:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006aefc2f9. 00000400:00000200:3.0:1784948903.540046:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948903.540051:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948903.540057:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948903.540067:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948903.540073:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3360:3360:242:4294967295] 192.168.202.137@tcp LPNI seq info [2176:2176:8:4294967295] 00000400:00000200:1.0:1784948903.540081:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948903.540090:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948903.540097:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948903.540100:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11176 health :700 00000800:00000010:1.0:1784948903.540123:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000400:00000200:3.0:1784948903.540124:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948903.540130:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000800:00000200:1.0:1784948903.540134:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784948903.540136:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000800:00000200:1.0:1784948903.540149:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948903.540155:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1784948903.540174:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000100:3.0:1784948903.540180:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:3.0:1784948903.540184:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003ed79484. 00000400:00000200:0.0:1784948903.540185:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000200:3.0:1784948903.540188:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948903.540194:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948903.540200:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000001:0.0:1784948903.540201:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948903.540204:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948903.540207:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:0.0:1784948903.540208:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:0.0:1784948903.540215:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:0.0:1784948903.540219:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948903.540222:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:0.0:1784948903.540231:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:0.0:1784948903.540235:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000467d4d37. 00000400:00000200:0.0:1784948903.540241:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:0.0:1784948903.540243:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:0.0:1784948903.540248:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:0.0:1784948903.540254:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:0.0:1784948903.540256:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948903.540257:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948903.540258:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:0.0:1784948903.540260:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:0.0:1784948903.540263:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948903.540264:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:0.0:1784948903.540267:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fff5e130. 00000400:00000200:0.0:1784948903.540269:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:0.0:1784948903.540271:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:0.0:1784948903.540273:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000800:00000200:2.0:1784948903.540384:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948903.540388:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000800:00000001:1.0:1784948903.540389:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948903.540393:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948903.540399:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784948903.540402:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:1.0:1784948903.540403:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948903.540406:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948903.540409:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948903.540420:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948903.540429:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948903.540452:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948903.540461:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948903.540470:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948903.540474:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3361:3361:244:4294967295] 192.168.202.137@tcp LPNI seq info [2177:2177:8:4294967295] 00000400:00000200:1.0:1784948903.540483:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948903.540487:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948903.540490:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000800:00000200:1.0:1784948903.540495:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948903.540500:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948903.540503:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948903.540538:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948903.540588:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948903.540593:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000400:00000200:1.0:1784948903.540600:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948903.540602:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948903.540608:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948903.540612:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948903.540616:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948903.540617:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948903.540625:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948903.540631:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948903.540645:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x871d 00000800:00000001:2.0:1784948903.540650:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948903.540654:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948903.540656:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948903.540659:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948903.540663:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:2.0:1784948903.540664:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000400:00000200:2.0:1784948903.540668:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948903.540670:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000090a03b90. 00000400:00000010:2.0:1784948903.540672:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000043d62367. 00000800:00000001:2.0:1784948903.540677:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948903.540679:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948904.179965:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948904.179971:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948904.179977:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948904.179978:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948904.180033:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948904.180037:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948904.180041:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948904.371992:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948904.372018:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948904.372020:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948904.372026:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948904.372029:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948904.372033:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948904.372035:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948904.372045:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948904.372053:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948904.372062:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086d6bb31. 00000020:00000040:2.0:1784948904.372069:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948904.372073:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948904.372076:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948904.372081:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948904.372091:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000635133da. 00000100:00000001:2.0:1784948904.372098:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948904.372107:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948904.372118:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948904.372120:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948904.372124:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086d6bb31] to pc [ptlrpcd_00_03+3] req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948904.372150:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948904.372153:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948904.372156:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948904.372159:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948904.372162:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948904.372165:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000078cad088. 00000020:00000040:2.0:1784948904.372167:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948904.372178:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948904.372180:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948904.372182:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784948904.372182:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.372189:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1784948904.372189:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00000001:1.0:1784948904.372194:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948904.372194:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948904.372204:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948904.372205:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948904.372242:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948904.372244:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948904.372245:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948904.372247:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948904.372252:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948904.372255:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948904.372255:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948904.372259:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000078cad088] to pc [ptlrpcd_00_00+0] req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948904.372260:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948904.372268:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948904.372276:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289660800:0@lo:400: 00000100:00000001:1.0:1784948904.372281:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948904.372286:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948904.372288:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948904.372290:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948904.372294:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948904.372296:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000035b54a67. 02000000:00000001:1.0:1784948904.372300:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948904.372302:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11181) 00000400:00000010:1.0:1784948904.372306:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019017dd3. 00000400:00000010:1.0:1784948904.372314:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000100:00000200:1.0:1784948904.372321:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289660800, portal 25 00000100:00000001:3.0:1784948904.372324:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948904.372325:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948904.372329:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470019712 : -115544239531904 : ffff96e9c2a59880) 00000100:00000001:3.0:1784948904.372330:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948904.372335:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948904.372339:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948904.372339:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948904.372345:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948904.372345:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948904.372367:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948904.372369:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948904.372372:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948904.372373:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784948904.372376:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:1.0:1784948904.372377:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 02000000:00000001:3.0:1784948904.372381:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1784948904.372382:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289660800, offset 0 02000000:00000001:3.0:1784948904.372388:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948904.372393:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:3.0:1784948904.372394:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289660928:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948904.372400:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948904.372403:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948904.372404:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948904.372406:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784948904.372411:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:3.0:1784948904.372411:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000898cc47a. 02000000:00000001:3.0:1784948904.372416:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948904.372422:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000400:00000200:1.0:1784948904.372428:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437d80 00000400:00000010:3.0:1784948904.372428:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000100:00000200:3.0:1784948904.372433:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289660928, portal 10 00000100:00000001:3.0:1784948904.372435:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948904.372436:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470018816 : -115544239532800 : ffff96e9c2a59500) 00000100:00000040:3.0:1784948904.372444:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948904.372450:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7d99 [2] + 0 00000400:00000010:1.0:1784948904.372468:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000179aef9d. 00000100:00000001:3.0:1784948904.372472:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948904.372477:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948904.372478:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000e71152c9 00000400:00000010:3.0:1784948904.372479:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000100:00000200:3.0:1784948904.372485:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289660928, offset 0 00000400:00000200:3.0:1784948904.372489:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948904.372490:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948904.372496:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948904.372501:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948904.372506:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948904.372511:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948904.372514:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948904.372517:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948904.372520:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3362:3362:244:4294967295] 192.168.202.137@tcp LPNI seq info [2178:2178:8:4294967295] 00000400:00000200:3.0:1784948904.372530:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948904.372535:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948904.372536:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e71152c9 00000400:00000010:1.0:1784948904.372540:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000e71152c9. 00000800:00000010:3.0:1784948904.372540:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000100:00000001:1.0:1784948904.372548:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:3.0:1784948904.372552:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:1.0:1784948904.372553:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784948904.372560:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000569fbc5c x1871651289660800 msgsize 224 00000800:00000200:3.0:1784948904.372569:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:1.0:1784948904.372570:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784948904.372577:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:3.0:1784948904.372577:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948904.372591:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948904.372609:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948904.372611:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:1.0:1784948904.372615:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000100:00000040:3.0:1784948904.372615:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2091 usecs (3 jiffies) 00000100:00000001:1.0:1784948904.372622:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948904.372623:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305843715 00000100:00000200:1.0:1784948904.372627:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948904.372631:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077930f7d. 00000100:00000001:3.0:1784948904.372641:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784948904.372642:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948904.372647:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948904.372654:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948904.372654:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305843715 00000100:00000001:1.0:1784948904.372658:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:3.0:1784948904.372660:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000068c34307. 00000100:00000001:1.0:1784948904.372665:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948904.372666:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948904.372672:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305843754 00000100:00000001:3.0:1784948904.372674:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305843715 00000100:00000010:3.0:1784948904.372680:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000001:1.0:1784948904.372685:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305843715 00000100:00000001:3.0:1784948904.372685:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948904.372692:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305843715 00000100:00000001:1.0:1784948904.372695:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305843754 00000100:00000010:3.0:1784948904.372696:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:2.0:1784948904.372699:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948904.372703:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948904.372705:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305843715 00000100:00100000:2.0:1784948904.372706:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289660800 00000100:00000001:3.0:1784948904.372708:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948904.372712:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305843754 02000000:00000001:2.0:1784948904.372714:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948904.372714:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.372719:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305843715 00000100:00000001:2.0:1784948904.372721:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948904.372722:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.372725:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305843754 00000100:00000001:3.0:1784948904.372728:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948904.372729:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.372731:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305843715 00000100:00000001:3.0:1784948904.372737:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948904.372738:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948904.372740:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948904.372742:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948904.372744:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.372764:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948904.372764:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948904.372768:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948904.372769:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948904.372773:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948904.372776:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289660800 00000100:00000001:1.0:1784948904.372778:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948904.372778:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948904.372780:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948904.372783:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948904.372787:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948904.372788:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1784948904.372795:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784948904.372796:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948904.372799:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784948904.372801:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948904.372804:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784948904.372809:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784948904.372812:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948904.372817:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948904.372822:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000001:2.0:1784948904.372825:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1784948904.372827:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:2.0:1784948904.372829:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000800:00000010:0.0:1784948904.372834:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000800:00000001:3.0:1784948904.372837:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784948904.372838:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000467d4d37. 00000400:00000200:0.0:1784948904.372841:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:2.0:1784948904.372846:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c335e9c8. 00000400:00000200:0.0:1784948904.372849:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948904.372850:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948904.372854:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000800:00000001:3.0:1784948904.372856:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948904.372857:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:0.0:1784948904.372862:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000800:00000001:3.0:1784948904.372864:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948904.372866:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948904.372867:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948904.372874:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000200:0.0:1784948904.372878:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948904.372878:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948904.372879:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948904.372884:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948904.372895:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437e00 00000100:00000001:0.0:1784948904.372896:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948904.372896:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948904.372939:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948904.372940:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 159408 00000100:00000001:2.0:1784948904.372946:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948904.372955:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948904.372958:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948904.372961:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1784948904.372965:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948904.372966:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948904.372970:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784948904.372972:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948904.372977:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784948904.372981:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:2.0:1784948904.372982:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 731 00000100:00000001:1.0:1784948904.372983:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:2.0:1784948904.372989:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948904.372990:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948904.372990:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948904.372994:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784948904.372996:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742037584 : -115538967514032 : ffff96eafce23450) 00000100:00000001:1.0:1784948904.372997:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948904.373001:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000e7e08a5. 00000100:00000040:3.0:1784948904.373005:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000000e7e08a5 x1871651289660928 msgsize 224 00000100:00000040:2.0:1784948904.373018:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000569fbc5c x1871651289660800/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:565/0 lens 224/0 e 0 to 0 dl 1784948915 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948904.373029:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784948904.373040:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948904.373043:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000001:3.0:1784948904.373045:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784948904.373046:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948904.373050:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000800:00000001:3.0:1784948904.373056:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784948904.373057:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000569fbc5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289660800:12345-0@lo:400: 00000800:00000001:3.0:1784948904.373062:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948904.373063:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:2.0:1784948904.373063:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289660800 00000100:00100000:1.0:1784948904.373068:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289660928 00000020:00000001:2.0:1784948904.373071:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:1.0:1784948904.373072:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948904.373076:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:2.0:1784948904.373077:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784948904.373078:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948904.373080:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948904.373083:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948904.373084:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948904.373088:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00100000:1.0:1784948904.373089:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289660928 00000020:00000001:2.0:1784948904.373092:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948904.373093:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:2.0:1784948904.373096:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784948904.373097:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784948904.373101:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948904.373102:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:2.0:1784948904.373105:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000040:1.0:1784948904.373107:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784948904.373109:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948904.373111:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948904.373112:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000001:1.0:1784948904.373116:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948904.373117:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948904.373121:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:2.0:1784948904.373122:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:1.0:1784948904.373129:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 02000000:00000010:2.0:1784948904.373130:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000001288f9. 00000100:00000001:1.0:1784948904.373135:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948904.373136:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948904.373141:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948904.373142:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000020:00000001:2.0:1784948904.373144:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948904.373148:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289660800 00000020:00000010:1.0:1784948904.373151:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dbce179e. 00010000:00000001:2.0:1784948904.373152:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000010:1.0:1784948904.373158:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002509680d. 00010000:00000200:2.0:1784948904.373159:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000569fbc5c x1871651289660800/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:565/0 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948904.373167:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00010000:00000001:2.0:1784948904.373173:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948904.373187:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948904.373189:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00001000:2.0:1784948904.373191:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=76 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948904.373196:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948904.373198:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784948904.373199:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:2.0:1784948904.373204:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948904.373205:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948904.373210:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784948904.373214:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000001:2.0:1784948904.373215:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784948904.373220:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:2.0:1784948904.373220:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.373225:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948904.373225:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948904.373230:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948904.373234:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000100:00000001:1.0:1784948904.373243:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948904.373254:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948904.373255:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:2.0:1784948904.373259:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289660800, offset 224 00000100:00100000:1.0:1784948904.373261:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 707 00000100:00000040:1.0:1784948904.373267:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000200:2.0:1784948904.373272:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948904.373273:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482523392 : -115544227028224 : ffff96e9c3646300) 00000100:00000040:1.0:1784948904.373281:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000e7e08a5 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:565/0 lens 224/0 e 0 to 0 dl 1784948915 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948904.373288:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784948904.373302:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948904.373305:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000400:00000200:2.0:1784948904.373307:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437d80 00000100:00000001:1.0:1784948904.373310:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948904.373316:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000400:00000200:2.0:1784948904.373320:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8721 [1] + 224 00000100:00100000:1.0:1784948904.373323:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000e7e08a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289660928:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948904.373334:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289660928 00000020:00000001:1.0:1784948904.373336:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:2.0:1784948904.373338:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784948904.373339:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948904.373342:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948904.373345:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948904.373349:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:2.0:1784948904.373350:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:1.0:1784948904.373352:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948904.373358:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948904.373359:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1784948904.373360:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948904.373365:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948904.373367:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948904.373370:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948904.373371:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948904.373375:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948904.373378:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948904.373382:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b01c6f1e. 02000000:00000001:1.0:1784948904.373387:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.373392:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948904.373395:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948904.373396:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1784948904.373399:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289660928 00010000:00000001:1.0:1784948904.373404:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948904.373411:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000e7e08a5 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:565/0 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948904.373428:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948904.373430:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948904.373432:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=71 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948904.373435:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948904.373437:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948904.373439:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948904.373441:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784948904.373442:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1784948904.373443:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.373448:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948904.373450:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948904.373453:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000200:2.0:1784948904.373455:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:2.0:1784948904.373459:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000100:00000001:0.0:1784948904.373472:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948904.373476:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948904.373476:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000200:1.0:1784948904.373477:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289660928, offset 224 00000100:00000001:2.0:1784948904.373482:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:0.0:1784948904.373485:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948904.373489:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784948904.373491:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948904.373492:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019017dd3. 00000400:00000200:0.0:1784948904.373499:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000100:00000040:2.0:1784948904.373503:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000010:0.0:1784948904.373504:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000100:00000001:0.0:1784948904.373510:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:2.0:1784948904.373512:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:1.0:1784948904.373515:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784948904.373517:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948904.373518:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948904.373526:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:1.0:1784948904.373532:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000040:2.0:1784948904.373537:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000569fbc5c x1871651289660800/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:565/0 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948904.373542:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948904.373547:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3363:3363:244:4294967295] 192.168.202.137@tcp LPNI seq info [2179:2179:8:4294967295] 00000400:00000200:1.0:1784948904.373580:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948904.373588:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948904.373592:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948904.373594:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000569fbc5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289660800:12345-0@lo:400: Request processed in 548us (1026us total) trans 0 rc 0/0 00000800:00000200:1.0:1784948904.373595:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948904.373597:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948904.373602:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000800:00000010:1.0:1784948904.373604:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000100:00000001:0.0:1784948904.373607:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948904.373611:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948904.373614:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948904.373618:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:2.0:1784948904.373619:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 731 00000100:00001000:0.0:1784948904.373622:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=76 v=5 (5 5 5 5) 00000100:00000040:2.0:1784948904.373628:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000800:00000200:1.0:1784948904.373629:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00001000:0.0:1784948904.373630:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=76 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948904.373637:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000200:1.0:1784948904.373638:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948904.373639:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948904.373644:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948904.373645:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00010000:00000001:0.0:1784948904.373648:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948904.373651:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948904.373653:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c335e9c8. 00000100:00000001:0.0:1784948904.373654:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948904.373655:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948904.373662:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948904.373665:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000020:00000010:2.0:1784948904.373673:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00010000:00000001:1.0:1784948904.373679:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948904.373683:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948904.373683:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000467d4d37. 00000100:00000001:0.0:1784948904.373684:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948904.373692:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948904.373693:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1784948904.373696:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948904.373697:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948904.373701:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000e7e08a5 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:565/0 lens 224/224 e 0 to 0 dl 1784948915 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948904.373708:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000040:0.0:1784948904.373709:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784948904.373714:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000001288f9. 02000000:00000001:2.0:1784948904.373724:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00100000:1.0:1784948904.373725:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000e7e08a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289660928:12345-192.168.202.137@tcp:400: Request processed in 416us (711us total) trans 0 rc 0/0 00000100:00000001:2.0:1784948904.373731:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948904.373734:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000086d6bb31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289660800:0@lo:400: 00000100:00100000:1.0:1784948904.373741:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 707 00000100:00000200:2.0:1784948904.373742:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948904.373759:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948904.373763:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948904.373768:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000086d6bb31 x1871651289660800/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948904.373776:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d5071549. 00000100:00000040:1.0:1784948904.373777:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948904.373778:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1784948904.373784:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:0.0:1784948904.373785:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000010:2.0:1784948904.373788:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000532cd756. 02000000:00000010:0.0:1784948904.373790:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000035b54a67. 00000100:00000001:1.0:1784948904.373790:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948904.373797:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002509680d. 02000000:00000001:0.0:1784948904.373800:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948904.373804:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:1.0:1784948904.373804:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000040:0.0:1784948904.373808:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000010:1.0:1784948904.373811:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dbce179e. 00000020:00000001:0.0:1784948904.373812:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948904.373817:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000635133da. 00000020:00000040:1.0:1784948904.373820:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 02000000:00000001:0.0:1784948904.373822:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948904.373826:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1784948904.373827:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1784948904.373838:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000086d6bb31. 00000100:00000001:0.0:1784948904.373845:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948904.373847:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948904.373849:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948904.373852:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948904.373854:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948904.373860:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948904.373860:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000800:00000001:2.0:1784948904.373861:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948904.373868:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784948904.373872:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948904.373873:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948904.373875:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948904.373883:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948904.373885:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948904.373889:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948904.373892:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:3.0:1784948904.373898:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000400:00000200:2.0:1784948904.373928:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948904.373941:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437e00 00000100:00000001:3.0:1784948904.373943:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948904.373947:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948904.373952:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8729 [1] + 224 00000800:00000001:2.0:1784948904.373961:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948904.373977:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948904.373981:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948904.373985:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948904.373990:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948904.373995:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948904.374006:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948904.374031:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948904.374037:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948904.374041:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948904.374076:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948904.374080:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948904.374084:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948904.374086:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948904.374089:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000400:00000200:3.0:1784948904.374095:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:3.0:1784948904.374096:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000100:00000001:3.0:1784948904.374098:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948904.374102:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948904.374110:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948904.374116:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948904.374117:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948904.374119:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948904.374120:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948904.374122:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948904.374123:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948904.374125:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948904.374129:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=71 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948904.374132:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=71 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948904.374134:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948904.374135:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948904.374136:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948904.374139:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948904.374145:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948904.374147:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948904.374148:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948904.374160:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948904.374161:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948904.374163:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948904.374169:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948904.374171:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948904.374172:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948904.374174:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948904.374186:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000078cad088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289660928:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948904.374190:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948904.374193:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948904.374196:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000078cad088 x1871651289660928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948904.374205:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948904.374207:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948904.374209:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000898cc47a. 02000000:00000001:3.0:1784948904.374213:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948904.374215:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948904.374226:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948904.374230:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948904.374232:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 02000000:00000001:3.0:1784948904.374235:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948904.374237:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948904.374239:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000078cad088. 00000100:00000001:3.0:1784948904.374242:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948904.374243:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948904.374246:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948904.374251:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948904.374252:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948904.374255:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948904.374257:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948904.414987:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305843754, at 4305843755 00000100:00000040:3.0:1784948904.414992:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305843754, at 4305843755 00000100:00000010:0.0:1784948904.414999:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077930f7d. 00000100:00000010:3.0:1784948904.415003:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:1.0:1784948904.415016:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305843754, at 4305843755 00000100:00000040:2.0:1784948904.415019:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305843754, at 4305843755 00000100:00000010:1.0:1784948904.415041:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000068c34307. 00000100:00000010:2.0:1784948904.415044:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000800:00000100:3.0:1784948904.564067:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000010:3.0:1784948904.564075:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000069a945c8. 00000400:00000200:3.0:1784948904.564087:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948904.564094:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948904.564101:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11177 health :900 00000400:00000200:3.0:1784948904.564108:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948904.564113:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:3.0:1784948904.564119:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:0.0:1784948904.564193:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948904.564206:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:0.0:1784948904.564215:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948904.564217:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948904.564220:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:0.0:1784948904.564223:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000400:00000200:0.0:1784948904.564229:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948904.564234:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:0.0:1784948904.564239:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dc7a25c6. 00000400:00000200:0.0:1784948904.564248:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:0.0:1784948904.564251:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:0.0:1784948904.564255:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00010000:00000001:2.0:1784948905.204043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948905.204054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948905.204063:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948905.204066:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948905.204204:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948905.204211:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948905.204215:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948905.396028:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948905.396044:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948905.396047:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948905.396054:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948905.396058:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948905.396063:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948905.396066:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948905.588079:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948905.588094:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000057de27a4. 00000400:00000010:1.0:1784948905.588107:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dbce179e. 00000400:00000010:1.0:1784948905.588117:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000400:00000010:1.0:1784948905.588130:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000098d04a76. 00000400:00000200:1.0:1784948905.588139:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948905.588160:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948905.588171:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948905.588178:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3364:3364:245:4294967295] 192.168.202.137@tcp LPNI seq info [2180:2180:8:4294967295] 00000400:00000200:1.0:1784948905.588190:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948905.588204:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948905.588215:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948905.588223:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000800:00000200:1.0:1784948905.588234:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948905.588248:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948905.588255:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000031deb17e type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948905.588287:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11179 health :900 00000800:00000200:3.0:1784948905.588454:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948905.588463:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:3.0:1784948905.588474:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948905.588482:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784948905.588489:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784948905.588493:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948905.588508:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948905.588520:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948905.588524:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948905.588529:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948905.588536:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948905.588549:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784948905.588560:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784948905.588573:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948905.588579:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3365:3365:245:4294967295] 192.168.202.137@tcp LPNI seq info [2181:2181:8:4294967295] 00000400:00000200:3.0:1784948905.588591:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784948905.588599:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948905.588604:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784948905.588612:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948905.588619:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948905.588623:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 3, nob 144 niov 1 nkiov 1 00000800:00000200:3.0:1784948905.588776:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948905.588791:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:3.0:1784948905.588801:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948905.588809:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948905.588809:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948905.588813:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948905.588835:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948905.588838:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948905.588849:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948905.588862:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948905.588877:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948905.588901:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x873d 00000800:00000001:0.0:1784948905.588941:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948905.588954:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948905.588957:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948905.588962:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948905.588969:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000400:00000010:0.0:1784948905.588973:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000400:00000200:0.0:1784948905.588977:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948905.588981:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000057de27a4. 00000400:00000010:0.0:1784948905.588985:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dbce179e. 00000800:00000001:0.0:1784948905.589007:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948905.589010:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948906.228052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948906.228063:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948906.228071:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948906.228072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948906.228206:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948906.228216:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948906.228222:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948906.420152:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948906.420189:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948906.420191:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948906.420200:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948906.420205:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948906.420209:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948906.420213:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948907.252057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948907.252067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948907.252074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948907.252076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948907.252204:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948907.252214:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948907.252219:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948907.444015:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948907.444031:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948907.444034:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948907.444039:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948907.444043:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948907.444063:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948907.444066:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948908.276039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948908.276067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948908.276075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948908.276076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948908.276179:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948908.276192:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948908.276199:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948908.468063:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948908.468078:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948908.468079:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948908.468085:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948908.468088:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948908.468092:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948908.468094:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784948908.660042:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.3@tcp 00000800:00000010:3.0:1784948908.660054:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000925a7f5a. 00000400:00000200:3.0:1784948908.660070:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948908.660086:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948908.660100:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11181 health :900 00000400:00000200:3.0:1784948908.660107:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948908.660112:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:3.0:1784948908.660117:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000800:00000100:3.0:1784948908.660150:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.9@tcp 00000800:00000010:3.0:1784948908.660153:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006d8f9877. 00000400:00000200:3.0:1784948908.660158:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948908.660163:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948908.660168:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948908.660172:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:3.0:1784948908.660179:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:0.0:1784948908.660188:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948908.660203:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 00000400:00000001:0.0:1784948908.660212:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948908.660215:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948908.660219:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:0.0:1784948908.660223:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000400:00000200:0.0:1784948908.660230:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948908.660236:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:0.0:1784948908.660241:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000043c3e1d8. 00000400:00000200:0.0:1784948908.660248:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-110 00000400:00000200:0.0:1784948908.660251:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -110 00000400:00000200:0.0:1784948908.660255:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00000400:00000200:0.0:1784948908.660260:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000001:0.0:1784948908.660263:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948908.660264:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948908.660266:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:0.0:1784948908.660268:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:0.0:1784948908.660272:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948908.660274:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:0.0:1784948908.660277:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000bbdbdf4. 00000400:00000200:0.0:1784948908.660281:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:0.0:1784948908.660283:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000400:00000200:0.0:1784948908.660287:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00010000:00000001:2.0:1784948909.300013:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948909.300024:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948909.300032:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948909.300033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948909.300143:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948909.300149:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948909.300152:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948909.492027:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:2.0:1784948909.492043:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948909.492045:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948909.492049:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948909.492055:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948909.492056:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784948909.492062:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948909.492066:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948909.492070:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948909.492070:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008d623923. 00000020:00000040:2.0:1784948909.492082:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948909.492089:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948909.492094:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948909.492102:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948909.492115:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003a8a98bc. 00000100:00000001:2.0:1784948909.492126:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.492140:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948909.492153:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948909.492156:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.492164:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d623923] to pc [ptlrpcd_00_01+1] req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.492195:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948909.492200:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948909.492205:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948909.492210:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948909.492216:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948909.492220:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b8226094. 00000020:00000040:2.0:1784948909.492222:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948909.492224:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948909.492225:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948909.492228:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948909.492232:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:2.0:1784948909.492235:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.492242:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948909.492249:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948909.492250:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.492255:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8226094] to pc [ptlrpcd_00_02+2] req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948909.492267:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948909.492279:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948909.492280:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948909.492290:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948909.492295:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11186) 00000100:00000040:1.0:1784948909.492307:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.492313:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948909.492324:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948909.492332:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948909.492337:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948909.492344:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.492345:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948909.492358:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948909.492363:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948909.492369:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948909.492373:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948909.492388:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948909.492393:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948909.492395:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.492396:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948909.492402:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948909.492405:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289661056:0@lo:400: 02000000:00000001:2.0:1784948909.492413:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948909.492415:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948909.492418:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948909.492428:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948909.492428:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948909.492441:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948909.492450:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289661184:192.168.202.137@tcp:400: 02000000:00000001:1.0:1784948909.492451:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948909.492460:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1784948909.492463:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000bf7eadde. 02000000:00000001:2.0:1784948909.492466:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948909.492470:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948909.492471:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948909.492492:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:1.0:1784948909.492497:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000034b95863. 02000000:00000010:2.0:1784948909.492499:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000467d4d37. 02000000:00000001:2.0:1784948909.492508:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948909.492522:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000010:2.0:1784948909.492522:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000037df9850. 00000400:00000010:2.0:1784948909.492543:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000100:00000200:1.0:1784948909.492546:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289661056, portal 25 00000100:00000001:1.0:1784948909.492554:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948909.492557:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470027776 : -115544239523840 : ffff96e9c2a5b800) 00000100:00000200:2.0:1784948909.492558:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289661184, portal 10 00000100:00000001:2.0:1784948909.492570:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948909.492575:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470022400 : -115544239529216 : ffff96e9c2a5a300) 00000100:00000040:1.0:1784948909.492576:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948909.492596:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948909.492598:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948909.492614:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948909.492624:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000001:2.0:1784948909.492631:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:1.0:1784948909.492635:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289661056, offset 0 00000100:00000040:2.0:1784948909.492641:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948909.492649:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000400:00000200:1.0:1784948909.492655:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000200:2.0:1784948909.492668:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289661184, offset 0 00000400:00000200:1.0:1784948909.492683:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784948909.492696:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948909.492742:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948909.492744:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437e80 00000400:00000200:2.0:1784948909.492755:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948909.492764:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7db5 [2] + 0 00000400:00000200:2.0:1784948909.492776:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000010:1.0:1784948909.492777:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006138467e. 00000400:00000200:2.0:1784948909.492787:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948909.492788:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000ebe1e3cd 00000400:00000200:2.0:1784948909.492798:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784948909.492803:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3366:3366:247:4294967295] 192.168.202.137@tcp LPNI seq info [2182:2182:8:4294967295] 00000400:00000200:1.0:1784948909.492805:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948909.492813:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948909.492823:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000400:00000200:1.0:1784948909.492832:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ebe1e3cd 00000800:00000200:2.0:1784948909.492832:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948909.492842:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000ebe1e3cd. 00000800:00000010:2.0:1784948909.492845:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000100:00000001:1.0:1784948909.492850:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948909.492854:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1784948909.492864:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:1.0:1784948909.492868:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000059bd87f4 x1871651289661056 msgsize 224 00000100:00100000:1.0:1784948909.492877:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000200:2.0:1784948909.492882:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:1.0:1784948909.492885:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:2.0:1784948909.492891:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948909.492963:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.492969:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2091 usecs (3 jiffies) 00000100:00000001:2.0:1784948909.492974:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305848836 00000100:00000001:1.0:1784948909.492975:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000010:2.0:1784948909.492982:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003b6d02be. 00000400:00000200:1.0:1784948909.492989:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000100:00000001:2.0:1784948909.492994:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000400:00000010:1.0:1784948909.492995:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:1.0:1784948909.493011:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948909.493013:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305848836 00000100:00000200:1.0:1784948909.493020:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784948909.493023:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e36e3817. 00000100:00000001:2.0:1784948909.493038:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948909.493052:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305848836 00000100:00000010:2.0:1784948909.493064:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fbfea82a. 00000100:00000001:1.0:1784948909.493065:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948909.493081:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000040:1.0:1784948909.493085:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.493094:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305848836 00000100:00000001:1.0:1784948909.493102:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948909.493104:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 00000800:00000200:0.0:1784948909.493106:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948909.493118:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948909.493122:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000010:0.0:1784948909.493124:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000100:00000001:2.0:1784948909.493124:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948909.493129:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.493140:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948909.493145:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948909.493146:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305848875 00000400:00000200:0.0:1784948909.493147:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948909.493159:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.493165:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948909.493168:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948909.493170:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305848836 00000100:00000001:2.0:1784948909.493173:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948909.493182:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:0.0:1784948909.493184:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000100:00000001:1.0:1784948909.493189:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305848875 00000100:00000001:2.0:1784948909.493190:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784948909.493191:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000100:00000001:2.0:1784948909.493199:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948909.493213:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948909.493214:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948909.493216:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305848836 00000100:00100000:3.0:1784948909.493223:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289661056 00000100:00000001:2.0:1784948909.493224:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948909.493226:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948909.493228:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305848875 02000000:00000001:3.0:1784948909.493240:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948909.493246:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305848836 00000100:00000001:3.0:1784948909.493249:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948909.493260:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948909.493264:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305848875 00000100:00000001:0.0:1784948909.493273:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784948909.493274:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948909.493275:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305848836 00000100:00100000:3.0:1784948909.493286:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289661056 00000100:00000040:0.0:1784948909.493288:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948909.493291:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948909.493301:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948909.493310:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948909.493312:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784948909.493324:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784948909.493327:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948909.493337:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948909.493337:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948909.493339:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948909.493359:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784948909.493361:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948909.493365:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948909.493373:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:3.0:1784948909.493374:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000001:0.0:1784948909.493380:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948909.493384:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948909.493389:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948909.493390:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:0.0:1784948909.493393:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948909.493395:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948909.493403:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784948909.493405:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000001:1.0:1784948909.493406:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948909.493428:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948909.493428:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948909.493447:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000400:00000200:0.0:1784948909.493461:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000010:3.0:1784948909.493465:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000898cc47a. 00000020:00000010:3.0:1784948909.493477:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a59da8f5. 00000400:00000200:0.0:1784948909.493479:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6437f00 00000020:00000010:3.0:1784948909.493503:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003c019a34. 00000400:00000200:0.0:1784948909.493515:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 159632 00000800:00000001:0.0:1784948909.493524:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784948909.493528:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000001:0.0:1784948909.493535:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948909.493536:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948909.493542:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784948909.493544:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948909.493547:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948909.493555:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948909.493561:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948909.493562:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948909.493564:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784948909.493572:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ee0ebd63. 00000100:00000040:0.0:1784948909.493575:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000ee0ebd63 x1871651289661184 msgsize 224 00000100:00100000:0.0:1784948909.493579:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784948909.493583:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948909.493599:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948909.493600:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948909.493605:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:0.0:1784948909.493609:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:3.0:1784948909.493619:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 732 00000800:00000001:0.0:1784948909.493620:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948909.493636:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784948909.493645:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00100000:2.0:1784948909.493646:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289661184 00000100:00000001:3.0:1784948909.493659:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742025296 : -115538967526320 : ffff96eafce20450) 02000000:00000001:2.0:1784948909.493660:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948909.493674:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784948909.493676:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000059bd87f4 x1871651289661056/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:570/0 lens 224/0 e 0 to 0 dl 1784948920 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.493683:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948909.493716:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948909.493726:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289661184 00000100:00000001:3.0:1784948909.493729:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:2.0:1784948909.493731:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948909.493735:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000010:3.0:1784948909.493735:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000020:00000001:2.0:1784948909.493747:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948909.493748:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1784948909.493758:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000010:3.0:1784948909.493758:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000020:00000001:2.0:1784948909.493779:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00100000:3.0:1784948909.493789:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000059bd87f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289661056:12345-0@lo:400: 00000020:00000001:2.0:1784948909.493791:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:3.0:1784948909.493808:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289661056 00000100:00000001:2.0:1784948909.493810:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:3.0:1784948909.493814:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000010:2.0:1784948909.493817:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000084bc9b0a. 00000020:00000001:3.0:1784948909.493821:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:2.0:1784948909.493823:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948909.493825:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948909.493828:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000084bc9b0a. 00000020:00000001:3.0:1784948909.493833:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948909.493836:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000010:2.0:1784948909.493838:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000020:00000001:3.0:1784948909.493850:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000010:2.0:1784948909.493852:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c335e9c8. 00000020:00000001:3.0:1784948909.493854:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948909.493862:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000010:2.0:1784948909.493864:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000020:00000001:3.0:1784948909.493866:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948909.493871:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948909.493876:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:2.0:1784948909.493880:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948909.493881:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948909.493889:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948909.493891:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948909.493903:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000010:3.0:1784948909.493905:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008d04782c. 02000000:00000001:3.0:1784948909.493957:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.493964:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948909.493965:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948909.493985:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948909.493991:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289661056 00000100:00000001:2.0:1784948909.493992:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948909.494003:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00010000:00000001:3.0:1784948909.494003:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784948909.494018:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784948909.494023:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000059bd87f4 x1871651289661056/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:570/0 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.494031:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784948909.494045:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948909.494050:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948909.494055:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=81 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948909.494056:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948909.494062:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948909.494064:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948909.494065:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1784948909.494078:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 708 00000100:00000001:0.0:1784948909.494079:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:3.0:1784948909.494085:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:2.0:1784948909.494093:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948909.494095:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:0.0:1784948909.494096:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.494104:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558293120 : -115544151258496 : ffff96e9c7e88a80) 00000100:00000001:0.0:1784948909.494106:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948909.494109:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948909.494115:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.494119:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ee0ebd63 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:570/0 lens 224/0 e 0 to 0 dl 1784948920 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948909.494126:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948909.494144:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784948909.494155:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:3.0:1784948909.494155:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000020:00000010:2.0:1784948909.494161:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000084bc9b0a. 00000100:00000200:3.0:1784948909.494164:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289661056, offset 224 00000100:00000001:2.0:1784948909.494166:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948909.494173:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000084bc9b0a. 00000400:00000200:3.0:1784948909.494181:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:2.0:1784948909.494186:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ee0ebd63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289661184:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784948909.494192:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289661184 00000020:00000001:2.0:1784948909.494195:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:3.0:1784948909.494201:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:2.0:1784948909.494203:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948909.494209:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948909.494214:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:3.0:1784948909.494223:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437e80 00000020:00000001:2.0:1784948909.494224:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784948909.494237:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:3.0:1784948909.494245:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8741 [1] + 224 00000020:00000001:2.0:1784948909.494247:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948909.494257:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948909.494261:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784948909.494262:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784948909.494270:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948909.494278:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:2.0:1784948909.494279:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948909.494293:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948909.494294:0:342757:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1784948909.494301:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000200:3.0:1784948909.494306:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784948909.494309:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000001288f9. 02000000:00000001:2.0:1784948909.494321:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.494329:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948909.494339:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948909.494353:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289661184 00000100:00000040:3.0:1784948909.494358:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948909.494361:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948909.494373:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ee0ebd63 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:570/0 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948909.494406:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948909.494409:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948909.494412:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=76 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948909.494417:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948909.494419:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948909.494423:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948909.494426:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000200:3.0:1784948909.494429:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 02000000:00000001:2.0:1784948909.494431:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948909.494445:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948909.494446:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000100:00000001:2.0:1784948909.494466:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948909.494467:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948909.494480:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000040:2.0:1784948909.494487:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948909.494492:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948909.494493:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000100:00000200:2.0:1784948909.494513:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289661184, offset 224 00000100:00000040:3.0:1784948909.494515:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:3.0:1784948909.494525:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784948909.494528:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948909.494529:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948909.494530:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948909.494537:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784948909.494540:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948909.494553:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000059bd87f4 x1871651289661056/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:570/0 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948909.494554:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948909.494564:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948909.494566:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000010:1.0:1784948909.494571:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000034b95863. 00000400:00000200:1.0:1784948909.494591:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000010:1.0:1784948909.494598:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000400:00000200:2.0:1784948909.494605:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3367:3367:247:4294967295] 192.168.202.137@tcp LPNI seq info [2183:2183:8:4294967295] 00000100:00100000:3.0:1784948909.494609:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000059bd87f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289661056:12345-0@lo:400: Request processed in 863us (1723us total) trans 0 rc 0/0 00000100:00000001:1.0:1784948909.494610:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948909.494625:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948909.494662:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 732 00000100:00000200:1.0:1784948909.494668:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948909.494670:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:2.0:1784948909.494676:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948909.494679:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948909.494685:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948909.494698:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a59da8f5. 00000020:00000010:3.0:1784948909.494728:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003c019a34. 00000800:00000200:2.0:1784948909.494741:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948909.494745:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784948909.494747:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000898cc47a. 00000100:00000001:1.0:1784948909.494749:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784948909.494750:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000062b4a223. 00000020:00000040:3.0:1784948909.494759:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784948909.494767:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948909.494770:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000800:00000200:2.0:1784948909.494776:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:1.0:1784948909.494777:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000010:3.0:1784948909.494779:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008d04782c. 00000100:00000001:1.0:1784948909.494789:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948909.494795:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948909.494796:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000800:00000200:2.0:1784948909.494807:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000001:1.0:1784948909.494809:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948909.494810:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948909.494824:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000062b4a223 type 1, nob 320 niov 1 nkiov 1 00000100:00001000:1.0:1784948909.494829:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=81 v=5 (5 5 5 5) 00000100:00000200:3.0:1784948909.494831:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:1.0:1784948909.494838:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=81 v=5 (1 1 1 1) 00000400:00000010:3.0:1784948909.494841:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000100:00000001:2.0:1784948909.494843:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948909.494844:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948909.494853:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948909.494861:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:3.0:1784948909.494863:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000449b49f2. 00000100:00000040:2.0:1784948909.494867:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948909.494869:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948909.494882:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948909.494882:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948909.494894:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948909.494937:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948909.494947:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ee0ebd63 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:570/0 lens 224/224 e 0 to 0 dl 1784948920 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948909.494958:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ee0ebd63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289661184:12345-192.168.202.137@tcp:400: Request processed in 803us (1380us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948909.494967:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 708 00000100:00000040:1.0:1784948909.494969:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948909.494971:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948909.494981:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948909.494986:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948909.494997:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c335e9c8. 00000100:00000001:1.0:1784948909.495001:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948909.495007:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784948909.495010:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948909.495016:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948909.495019:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000800:00000200:0.0:1784948909.495020:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:1.0:1784948909.495028:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1784948909.495029:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000062b4a223. 00000020:00000040:2.0:1784948909.495032:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000400:00000200:0.0:1784948909.495038:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948909.495044:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948909.495047:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948909.495069:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948909.495071:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948909.495085:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948909.495092:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000100:00100000:1.0:1784948909.495095:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289661056:0@lo:400: 00000800:00000001:3.0:1784948909.495097:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948909.495098:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000100:00000001:0.0:1784948909.495121:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948909.495124:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948909.495131:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948909.495134:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784948909.495142:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:1.0:1784948909.495143:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008d623923 x1871651289661056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948909.495159:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948909.495164:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948909.495168:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000bf7eadde. 00000400:00000200:3.0:1784948909.495169:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6437f00 02000000:00000001:1.0:1784948909.495177:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948909.495181:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948909.495186:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:3.0:1784948909.495187:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8745 [1] + 224 00000020:00000001:1.0:1784948909.495192:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948909.495199:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003a8a98bc. 00000800:00000001:3.0:1784948909.495201:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784948909.495204:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948909.495209:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948909.495211:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008d623923. 00000800:00000001:3.0:1784948909.495216:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948909.495218:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948909.495224:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948909.495225:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948909.495230:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948909.495238:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948909.495239:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948909.495244:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948909.495245:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784948909.495249:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948909.495254:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948909.495258:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948909.495275:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948909.495299:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948909.495307:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948909.495311:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948909.495325:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948909.495330:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948909.495335:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948909.495337:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948909.495340:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000037df9850. 00000400:00000200:2.0:1784948909.495345:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:2.0:1784948909.495346:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000100:00000001:2.0:1784948909.495349:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948909.495354:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948909.495363:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.495369:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948909.495370:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948909.495372:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948909.495374:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948909.495376:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948909.495377:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948909.495379:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948909.495385:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=76 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948909.495388:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=76 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948909.495390:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948909.495391:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948909.495393:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948909.495396:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948909.495402:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.495405:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948909.495406:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948909.495407:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948909.495409:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948909.495411:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.495416:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948909.495418:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948909.495420:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948909.495422:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948909.495427:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289661184:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948909.495431:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948909.495434:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948909.495436:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b8226094 x1871651289661184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948909.495441:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948909.495444:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948909.495445:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000467d4d37. 02000000:00000001:2.0:1784948909.495448:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948909.495449:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948909.495450:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948909.495452:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948909.495454:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:2.0:1784948909.495457:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948909.495459:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948909.495460:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b8226094. 00000100:00000001:2.0:1784948909.495463:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948909.495464:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948909.495466:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948909.495469:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948909.495470:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948909.495472:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948909.495474:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948909.536019:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305848875, at 4305848876 00000100:00000040:2.0:1784948909.536020:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305848875, at 4305848876 00000100:00000040:3.0:1784948909.536040:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305848875, at 4305848876 00000100:00000010:2.0:1784948909.536049:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fbfea82a. 00000100:00000010:1.0:1784948909.536054:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e36e3817. 00000100:00000010:3.0:1784948909.536061:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000040:0.0:1784948909.536102:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305848875, at 4305848876 00000100:00000010:0.0:1784948909.536115:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003b6d02be. 00000800:00000100:3.0:1784948909.684071:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.7@tcp 00000800:00000010:3.0:1784948909.684079:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000230b350e. 00000400:00000200:3.0:1784948909.684088:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948909.684096:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948909.684107:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948909.684114:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:3.0:1784948909.684128:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:1.0:1784948909.684194:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:3.0:1784948909.684196:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000800:00000010:3.0:1784948909.684204:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d5a127c9. 00000400:00000010:1.0:1784948909.684207:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a2236612. 00000400:00000200:3.0:1784948909.684209:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948909.684217:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948909.684221:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948909.684232:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000010:1.0:1784948909.684235:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf7eadde. 00000400:00000010:1.0:1784948909.684245:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000200:3.0:1784948909.684247:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:1.0:1784948909.684262:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c521f738. 00000400:00000200:1.0:1784948909.684273:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948909.684283:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784948909.684294:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948909.684304:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948909.684310:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3368:3368:249:4294967295] 192.168.202.137@tcp LPNI seq info [2184:2184:8:4294967295] 00000400:00000200:1.0:1784948909.684320:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948909.684332:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948909.684340:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948909.684348:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000800:00000010:1.0:1784948909.684353:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000400:00000001:2.0:1784948909.684359:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948909.684364:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1784948909.684375:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784948909.684375:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:2.0:1784948909.684389:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000400:00000200:2.0:1784948909.684400:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000200:1.0:1784948909.684402:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948909.684414:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784948909.684421:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:2.0:1784948909.684428:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000061851253. 00000400:00000200:2.0:1784948909.684437:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-110 00000400:00000200:2.0:1784948909.684440:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -110 00000400:00000200:1.0:1784948909.684446:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11183 health :800 00000400:00000200:2.0:1784948909.684458:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:2.0:1784948909.684467:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:2.0:1784948909.684473:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948909.684476:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948909.684479:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:2.0:1784948909.684480:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000400:00000200:2.0:1784948909.684486:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948909.684489:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:2.0:1784948909.684492:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001879354a. 00000400:00000200:2.0:1784948909.684497:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:2.0:1784948909.684499:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:2.0:1784948909.684507:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000800:00000200:0.0:1784948909.684633:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948909.684640:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000800:00000001:2.0:1784948909.684643:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948909.684649:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948909.684658:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948909.684662:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784948909.684663:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948909.684671:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948909.684674:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784948909.684684:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948909.684685:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:2.0:1784948909.684728:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948909.684739:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948909.684752:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948909.684758:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3369:3369:249:4294967295] 192.168.202.137@tcp LPNI seq info [2185:2185:8:4294967295] 00000400:00000200:2.0:1784948909.684767:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948909.684772:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948909.684777:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000036e798e7. 00000800:00000200:2.0:1784948909.684783:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948909.684789:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948909.684792:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000036e798e7 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784948909.684863:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948909.684939:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948909.684943:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784948909.684943:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000036e798e7. 00000800:00000001:3.0:1784948909.684951:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948909.684952:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948909.684958:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948909.684976:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948909.684980:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948909.684987:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948909.684999:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948909.685016:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x875d 00000800:00000001:0.0:1784948909.685023:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948909.685037:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948909.685040:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948909.685045:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948909.685052:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000010:0.0:1784948909.685055:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000400:00000200:0.0:1784948909.685059:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948909.685063:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a2236612. 00000400:00000010:0.0:1784948909.685067:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf7eadde. 00000800:00000001:0.0:1784948909.685074:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948909.685078:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948910.324021:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948910.324032:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948910.324041:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948910.324046:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948910.324049:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948910.324053:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948910.324056:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948910.324058:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948910.324060:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948910.324062:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948910.324185:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948910.324190:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948910.324194:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948910.516047:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948910.516072:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948910.516076:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948910.516086:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948910.516092:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948910.516099:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948910.516105:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784948910.708096:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.11@tcp 00000800:00000010:3.0:1784948910.708105:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c03b4f3a. 00000400:00000200:3.0:1784948910.708116:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948910.708123:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948910.708134:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11183 health :900 00000400:00000200:3.0:1784948910.708140:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948910.708145:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:3.0:1784948910.708151:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:2.0:1784948910.708224:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948910.708242:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:2.0:1784948910.708254:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948910.708257:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948910.708261:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000400:00000010:2.0:1784948910.708265:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000400:00000200:2.0:1784948910.708273:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948910.708280:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:2.0:1784948910.708286:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000208850ab. 00000400:00000200:2.0:1784948910.708295:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:2.0:1784948910.708298:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:2.0:1784948910.708301:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00010000:00000001:2.0:1784948911.348058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948911.348071:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948911.348082:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948911.348084:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948911.348220:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948911.348228:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948911.348233:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948911.540034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948911.540050:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948911.540054:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948911.540064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948911.540068:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948911.540073:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948911.540076:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948911.732061:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948911.732078:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a2236612. 00000800:00000100:3.0:1784948911.732078:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000800:00000010:3.0:1784948911.732095:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000057c7ab4. 00000400:00000010:1.0:1784948911.732102:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a17860fa. 00000400:00000200:3.0:1784948911.732107:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:1.0:1784948911.732114:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000400:00000200:3.0:1784948911.732126:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784948911.732139:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000dc3e31db. 00000400:00000200:3.0:1784948911.732145:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11184 health :800 00000400:00000200:1.0:1784948911.732159:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948911.732173:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948911.732182:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:1.0:1784948911.732194:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948911.732204:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948911.732210:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3370:3370:250:4294967295] 192.168.202.137@tcp LPNI seq info [2186:2186:8:4294967295] 00000400:00000200:1.0:1784948911.732221:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948911.732231:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948911.732245:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948911.732247:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000800:00000010:1.0:1784948911.732259:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000800:00000200:1.0:1784948911.732274:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948911.732288:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948911.732295:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784948911.732345:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948911.732364:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:2.0:1784948911.732376:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948911.732380:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948911.732384:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:2.0:1784948911.732388:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:2.0:1784948911.732398:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948911.732405:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:2.0:1784948911.732412:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007a6587e5. 00000400:00000200:2.0:1784948911.732424:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:2.0:1784948911.732428:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:2.0:1784948911.732433:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00000800:00000200:3.0:1784948911.732485:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948911.732491:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000800:00000001:2.0:1784948911.732495:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948911.732500:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948911.732509:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784948911.732515:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:2.0:1784948911.732520:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948911.732521:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784948911.732532:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948911.732544:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948911.732558:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948911.732577:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948911.732591:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948911.732605:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948911.732612:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3371:3371:251:4294967295] 192.168.202.137@tcp LPNI seq info [2187:2187:8:4294967295] 00000400:00000200:2.0:1784948911.732623:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948911.732629:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948911.732635:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000036e798e7. 00000800:00000200:2.0:1784948911.732642:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948911.732649:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948911.732654:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000036e798e7 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948911.732803:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948911.732893:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948911.732898:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000036e798e7. 00000400:00000200:2.0:1784948911.732935:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948911.732946:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948911.732950:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948911.732959:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948911.732970:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948911.732973:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948911.732982:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948911.732995:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948911.733017:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8761 00000800:00000001:0.0:1784948911.733025:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948911.733036:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948911.733039:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948911.733044:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948911.733051:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:0.0:1784948911.733054:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000400:00000200:0.0:1784948911.733058:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948911.733063:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a2236612. 00000400:00000010:0.0:1784948911.733067:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a17860fa. 00000800:00000001:0.0:1784948911.733075:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948911.733078:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948912.372048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948912.372063:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948912.372076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948912.372079:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948912.372197:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948912.372204:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948912.372208:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948912.564004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948912.564021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948912.564024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948912.564031:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948912.564035:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948912.564039:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948912.564042:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784948912.756099:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.26@tcp 00000800:00000010:3.0:1784948912.756107:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009ff40a64. 00000400:00000200:3.0:1784948912.756121:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948912.756129:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.26@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948912.756139:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11185 health :900 00000400:00000200:3.0:1784948912.756145:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948912.756150:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.26@tcp: 1 00000400:00000200:3.0:1784948912.756156:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.26@tcp: -114 00000800:00000100:3.0:1784948912.756191:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000800:00000010:3.0:1784948912.756195:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007b2ad4bf. 00000400:00000200:3.0:1784948912.756199:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948912.756210:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948912.756217:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948912.756224:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:2.0:1784948912.756232:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948912.756233:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:2.0:1784948912.756256:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.26@tcp(00000000ba34905a) state 0x840 00000400:00000001:2.0:1784948912.756270:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.756273:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.756278:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784948912.756282:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000400:00000200:2.0:1784948912.756291:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.756298:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.26@tcp: -114 00000400:00000010:2.0:1784948912.756304:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003f3c01c8. 00000400:00000200:2.0:1784948912.756314:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.26@tcp:-110 00000400:00000200:2.0:1784948912.756317:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.26@tcp(00000000ba34905a) state 0x2040 rc -110 00000400:00000200:2.0:1784948912.756321:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.26@tcp 00000400:00000200:2.0:1784948912.756341:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:2.0:1784948912.756345:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.756346:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.756348:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784948912.756350:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000400:00000200:2.0:1784948912.756353:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.756355:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:2.0:1784948912.756358:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009802f36f. 00000400:00000200:2.0:1784948912.756363:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:2.0:1784948912.756366:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:2.0:1784948912.756370:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:3.0:1784948912.756408:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.26@tcp NID 111.222.142.26@tcp: -110. discovery complete 00000400:00000200:3.0:1784948912.756416:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.142.26@tcp primary NID 111.222.142.26@tcp rc -110 00000100:00000010:3.0:1784948912.756438:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000480fb190. 00000100:00000001:3.0:1784948912.756469:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784948912.756473:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000480fb190 refcount 1 to 111.222.142.26@tcp 00010000:00000010:3.0:1784948912.756478:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000099f45de2. 00010000:00080000:3.0:1784948912.756531:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.142.26@tcp at tail 00010000:00000001:3.0:1784948912.756535:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948912.756540:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948912.756554:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 000000009af7b278. 00000100:00000400:3.0:1784948912.756560:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.173.27@tcp: rc = -75 00000100:00000040:3.0:1784948912.756564:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.173.27@tcp cnt=32 00000400:00000200:3.0:1784948912.756567:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784948912.756571:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784948912.756574:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784948912.756577:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784948912.756579:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784948912.756586:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784948912.756592:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.756596:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.756599:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784948912.756601:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784948912.756606:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948912.756628:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784948912.756631:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.756633:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.756636:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.756638:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784948912.756640:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784948912.756644:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948912.756652:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:3.0:1784948912.756655:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.756657:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.756661:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.756665:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784948912.756668:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:2.0:1784948912.756672:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948912.756674:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.756703:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784948912.756724:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000010:2.0:1784948912.756727:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000467d4d37. 00000400:00000200:3.0:1784948912.756729:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.756738:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784948912.756743:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:3.0:1784948912.756750:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.756756:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784948912.756758:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000010:2.0:1784948912.756762:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003e1fe5c5. 00000400:00000200:3.0:1784948912.756767:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.756780:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784948912.756785:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:3.0:1784948912.756789:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.756794:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.756799:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.756801:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784948912.756803:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784948912.756806:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.756817:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:2.0:1784948912.756826:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.756833:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.756838:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.756842:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:2.0:1784948912.756847:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3372:3372:253:4294967295] 111.222.173.1@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:2.0:1784948912.756856:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:2.0:1784948912.756865:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.756879:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784948912.756885:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784948912.756889:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948912.756890:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000036e798e7. 00000400:00000200:3.0:1784948912.756895:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.756901:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000800:00000200:2.0:1784948912.756951:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000200:2.1:1784948912.756961:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:2.0:1784948912.756969:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:3.0:1784948912.756974:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:2.0:1784948912.756975:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:3.0:1784948912.756990:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000200:2.0:1784948912.756999:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784948912.757009:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948912.757011:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000200:3.0:1784948912.757026:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000400:00000010:2.0:1784948912.757033:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000200:3.0:1784948912.757033:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.757046:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.757053:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.757057:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000010:2.0:1784948912.757059:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000028b23cff. 00000400:00000200:3.0:1784948912.757064:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:3.0:1784948912.757081:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.757082:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:2.0:1784948912.757104:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:2.0:1784948912.757110:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.757113:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.757115:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.757118:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:2.0:1784948912.757122:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3373:3373:252:4294967295] 111.222.173.2@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:2.0:1784948912.757126:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:2.0:1784948912.757132:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.757137:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:2.0:1784948912.757141:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d104522b. 00000400:00000200:3.0:1784948912.757142:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784948912.757154:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.757158:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948912.757164:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784948912.757177:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.757190:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000800:00000200:2.1:1784948912.757190:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784948912.757207:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:3.0:1784948912.757212:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948912.757217:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:3.0:1784948912.757220:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948912.757227:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:3.0:1784948912.757230:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948912.757234:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:3.0:1784948912.757245:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.757249:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784948912.757251:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000200:2.0:1784948912.757258:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000010:2.0:1784948912.757273:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000200:3.0:1784948912.757274:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948912.757291:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000200:3.0:1784948912.757299:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:3.0:1784948912.757302:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948912.757308:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000284efd3d. 00000400:00000200:3.0:1784948912.757308:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948912.757334:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784948912.757336:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.757350:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784948912.757356:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:2.0:1784948912.757360:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:3.0:1784948912.757366:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.757376:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.757383:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.757385:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.757389:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:2.0:1784948912.757393:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3374:3374:251:4294967295] 111.222.142.2@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:2.0:1784948912.757397:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:2.0:1784948912.757402:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.757408:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000800:00000010:2.0:1784948912.757416:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002a753d9d. 00000400:00000200:3.0:1784948912.757423:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000200:3.0:1784948912.757429:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784948912.757435:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784948912.757438:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948912.757448:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784948912.757458:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.757469:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000800:00000200:2.1:1784948912.757481:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784948912.757486:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000800:00000200:2.1:1784948912.757491:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784948912.757499:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000800:00000100:2.0:1784948912.757509:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.2@tcp 00000800:00000010:2.0:1784948912.757513:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002a753d9d. 00000400:00000200:3.0:1784948912.757515:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:3.0:1784948912.757526:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948912.757530:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948912.757531:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.757548:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:2.0:1784948912.757555:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948912.757564:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:2.0:1784948912.757573:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948912.757574:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:2.0:1784948912.757605:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:3.0:1784948912.757607:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.757642:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:2.0:1784948912.757657:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:2.0:1784948912.757662:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 rc 0 00000400:00000200:3.0:1784948912.757669:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784948912.757695:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.757697:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.757702:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.757705:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:3.0:1784948912.757708:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:2.0:1784948912.757713:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784948912.757721:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948912.757724:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000bbdbdf4. 00000400:00000010:2.0:1784948912.757731:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000400:00000200:3.0:1784948912.757732:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:3.0:1784948912.757744:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948912.757753:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004bff3bbc. 00000400:00000200:3.0:1784948912.757755:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.757773:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:2.0:1784948912.757779:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784948912.757947:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:3.0:1784948912.757952:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:2.0:1784948912.757958:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:3.0:1784948912.757969:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.757977:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.757985:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.757989:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.757995:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:2.0:1784948912.758000:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3375:3375:251:4294967295] 111.222.173.3@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:2.0:1784948912.758008:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:2.0:1784948912.758020:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.758032:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:2.0:1784948912.758038:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002a753d9d. 00000400:00000200:3.0:1784948912.758040:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784948912.758055:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784948912.758067:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784948912.758067:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.1:1784948912.758094:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784948912.758095:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.758115:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:2.0:1784948912.758121:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784948912.758130:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:2.0:1784948912.758137:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:3.0:1784948912.758149:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.758155:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:3.0:1784948912.758164:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784948912.758167:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.758173:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784948912.758174:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009802f36f. 00000400:00000200:3.0:1784948912.758186:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000010:2.0:1784948912.758187:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000200:3.0:1784948912.758208:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000010:2.0:1784948912.758211:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000573e3d89. 00000400:00000200:3.0:1784948912.758219:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:2.0:1784948912.758228:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784948912.758232:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.758245:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:2.0:1784948912.758250:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.758253:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.758255:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.758259:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:2.0:1784948912.758262:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3376:3376:250:4294967295] 111.222.142.3@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:2.0:1784948912.758267:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:2.0:1784948912.758274:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.758283:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:2.0:1784948912.758286:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008f5595bf. 00000400:00000200:3.0:1784948912.758294:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000400:00000200:3.0:1784948912.758303:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784948912.758310:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784948912.758313:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.758341:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000800:00000200:2.1:1784948912.758342:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784948912.758364:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:2.0:1784948912.758370:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:3.0:1784948912.758379:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:2.0:1784948912.758389:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x261 rc 0 00000400:00000200:3.0:1784948912.758398:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.758410:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784948912.758416:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784948912.758419:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.758424:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784948912.758427:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000061851253. 00000400:00000200:3.0:1784948912.758433:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000010:2.0:1784948912.758440:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000400:00000200:3.0:1784948912.758442:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000010:2.0:1784948912.758454:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000078e6b4ca. 00000400:00000200:3.0:1784948912.758455:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:3.0:1784948912.758496:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.758498:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:2.0:1784948912.758520:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:2.0:1784948912.758523:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.758526:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.758528:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.758531:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:2.0:1784948912.758535:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3377:3377:249:4294967295] 111.222.173.4@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:2.0:1784948912.758539:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:2.0:1784948912.758545:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.758549:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000800:00000010:2.0:1784948912.758553:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009ff40a64. 00000400:00000200:3.0:1784948912.758554:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:3.0:1784948912.758568:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784948912.758579:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784948912.758579:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.758599:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000800:00000200:2.1:1784948912.758603:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784948912.758613:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:3.0:1784948912.758617:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:3.0:1784948912.758622:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948912.758627:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:3.0:1784948912.758629:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.758631:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948912.758642:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:3.0:1784948912.758644:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:2.0:1784948912.758661:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784948912.758662:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784948912.758698:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:2.0:1784948912.758703:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000200:3.0:1784948912.758715:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948912.758721:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 00000400:00000200:3.0:1784948912.758728:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000010:2.0:1784948912.758729:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000200:3.0:1784948912.758739:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948912.758749:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000006233a0b. 00000400:00000200:3.0:1784948912.758751:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948912.758777:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:3.0:1784948912.758779:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.758796:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:3.0:1784948912.758800:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:2.0:1784948912.758805:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:3.0:1784948912.758811:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.758822:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.758830:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.758834:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.758838:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:2.0:1784948912.758842:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3378:3378:248:4294967295] 111.222.142.4@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:2.0:1784948912.758847:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:2.0:1784948912.758853:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.758864:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:2.0:1784948912.758869:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000057c7ab4. 00000400:00000200:3.0:1784948912.758873:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:3.0:1784948912.758883:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.758890:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948912.758898:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:3.0:1784948912.758938:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.758943:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:3.0:1784948912.758948:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:3.0:1784948912.758952:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000800:00000200:2.0:1784948912.758958:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784948912.758970:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000800:00000200:2.1:1784948912.758982:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:3.0:1784948912.758984:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.758992:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.758998:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759001:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:3.0:1784948912.759003:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:3.0:1784948912.759007:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948912.759021:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000800:00000200:2.1:1784948912.759023:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784948912.759028:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.759048:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000100:2.0:1784948912.759057:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.4@tcp 00000800:00000010:2.0:1784948912.759061:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000057c7ab4. 00000400:00000200:2.0:1784948912.759067:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948912.759073:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948912.759119:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759124:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:3.0:1784948912.759127:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:2.0:1784948912.759129:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948912.759134:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.759150:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:3.0:1784948912.759151:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:3.0:1784948912.759161:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948912.759175:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:3.0:1784948912.759185:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.759192:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759195:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:3.0:1784948912.759198:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:3.0:1784948912.759206:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.759211:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:3.0:1784948912.759218:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:2.0:1784948912.759225:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 rc 0 00000400:00000200:3.0:1784948912.759226:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.759232:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.759240:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759244:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000200:3.0:1784948912.759248:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000200:2.0:1784948912.759255:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000200:3.0:1784948912.759255:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948912.759271:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784948912.759274:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.759278:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.759280:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759283:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000010:2.0:1784948912.759285:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000244a9cae. 00000400:00000200:3.0:1784948912.759287:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:3.0:1784948912.759297:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948912.759304:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000400:00000200:3.0:1784948912.759306:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:3.0:1784948912.759312:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.759323:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.759327:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759331:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000010:2.0:1784948912.759334:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005f31e551. 00000400:00000200:3.0:1784948912.759335:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:3.0:1784948912.759364:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.759365:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:3.0:1784948912.759380:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:3.0:1784948912.759383:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.759387:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.759390:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759393:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:3.0:1784948912.759395:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:2.0:1784948912.759405:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:3.0:1784948912.759407:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948912.759426:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.759431:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.759434:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.759438:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:2.0:1784948912.759443:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3379:3379:248:4294967295] 111.222.173.5@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:2.0:1784948912.759451:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:2.0:1784948912.759462:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.759476:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000400:00000200:3.0:1784948912.759480:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784948912.759490:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948912.759501:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000057c7ab4. 00000400:00000200:3.0:1784948912.759501:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948912.759513:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759517:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784948912.759528:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000800:00000200:2.0:1784948912.759537:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:3.0:1784948912.759543:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948912.759555:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000200:3.0:1784948912.759561:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948912.759563:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.1:1784948912.759574:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:3.0:1784948912.759575:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948912.759590:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000100:00000010:3.0:1784948912.759599:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 000000009af7b278. 00000400:00000200:2.0:1784948912.759605:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000020:00000001:3.0:1784948912.759617:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000200:2.0:1784948912.759625:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x261 rc 0 00010000:00000001:3.0:1784948912.759626:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000100:00000001:3.0:1784948912.759633:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000010:3.0:1784948912.759645:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000898cc47a. 00000400:00000200:2.0:1784948912.759646:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000200:3.0:1784948912.759663:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000898cc47a nid 111.222.173.27@tcp 00000400:00000010:3.0:1784948912.759672:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000ead0e753. 00000400:00000010:2.0:1784948912.759673:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000033c25eed. 00000400:00000200:3.0:1784948912.759700:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 00000000ead0e753 net tcp 00000400:00000010:3.0:1784948912.759713:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000b8810b40. 00000400:00000010:2.0:1784948912.759717:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000492bb2ef. 00000400:00000200:3.0:1784948912.759725:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000b8810b40 nid 111.222.173.27@tcp 00000400:00000010:2.0:1784948912.759734:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c92a48c0. 00000400:00000200:3.0:1784948912.759747:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.173.27@tcp NID 111.222.173.27@tcp flags 0x0 00000400:00000200:3.0:1784948912.759755:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.173.27@tcp: 0 00000400:00000200:2.0:1784948912.759762:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:3.0:1784948912.759768:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.27@tcp: 0 00000400:00000200:3.0:1784948912.759773:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948912.759789:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:2.0:1784948912.759800:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.759809:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.759819:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.759825:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:2.0:1784948912.759838:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3380:3380:247:4294967295] 111.222.142.5@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:2.0:1784948912.759851:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:2.0:1784948912.759868:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.759878:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:2.0:1784948912.759885:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003a8a98bc. 00000800:00000200:2.0:1784948912.759896:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784948912.759970:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:2.0:1784948912.759996:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:2.0:1784948912.760003:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:2.0:1784948912.760011:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:2.0:1784948912.760020:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001a4eba99. 00000400:00000010:2.0:1784948912.760027:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000010:2.0:1784948912.760037:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003dae5c32. 00000400:00000200:2.0:1784948912.760045:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:2.0:1784948912.760058:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:2.0:1784948912.760064:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.760069:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.760073:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.760079:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:2.0:1784948912.760084:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3381:3381:246:4294967295] 111.222.173.6@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:2.0:1784948912.760093:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:2.0:1784948912.760103:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.760111:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:2.0:1784948912.760117:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000062b4a223. 00000800:00000200:2.0:1784948912.760127:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784948912.760136:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:2.0:1784948912.760167:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:2.0:1784948912.760171:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:2.0:1784948912.760180:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000001:2.0:1784948912.760186:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.760190:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.760194:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:2.0:1784948912.760197:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:2.0:1784948912.760204:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.760217:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:2.0:1784948912.760225:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000400:00000200:2.0:1784948912.760234:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-5 00000400:00000200:2.0:1784948912.760245:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -5 00000400:00000200:2.0:1784948912.760252:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:2.0:1784948912.760265:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:2.0:1784948912.760274:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000010:2.0:1784948912.760281:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000010:2.0:1784948912.760291:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001b14ffb3. 00000400:00000200:2.0:1784948912.760299:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:2.0:1784948912.760309:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:2.0:1784948912.760314:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.760321:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.760327:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.760334:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:2.0:1784948912.760341:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3382:3382:245:4294967295] 111.222.142.6@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:2.0:1784948912.760351:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:2.0:1784948912.760361:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.760370:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:2.0:1784948912.760376:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000034aacccc. 00000800:00000200:2.0:1784948912.760385:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.0:1784948912.760389:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 14 secs later) 00000800:00000200:2.1:1784948912.760399:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784948912.760404:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 14 secs later) 00000800:00000100:2.0:1784948912.760411:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.6@tcp 00000800:00000010:2.0:1784948912.760415:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000034aacccc. 00000400:00000200:2.0:1784948912.760421:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948912.760428:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948912.760434:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948912.760439:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:2.0:1784948912.760444:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:2.0:1784948912.760454:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:2.0:1784948912.760458:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 rc 0 00000400:00000200:2.0:1784948912.760464:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:2.0:1784948912.760473:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf0e6e18. 00000400:00000010:2.0:1784948912.760478:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000400:00000010:2.0:1784948912.760484:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fce68683. 00000400:00000200:2.0:1784948912.760490:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:2.0:1784948912.760499:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:2.0:1784948912.760504:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.760508:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.760514:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.760519:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:2.0:1784948912.760525:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3383:3383:245:4294967295] 111.222.142.7@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:2.0:1784948912.760534:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:2.0:1784948912.760545:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.760553:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:2.0:1784948912.760558:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000034aacccc. 00000800:00000200:2.0:1784948912.760566:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784948912.760574:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:2.0:1784948912.760580:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:2.0:1784948912.760584:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:2.0:1784948912.760591:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:2.0:1784948912.760597:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c73b3f00. 00000400:00000010:2.0:1784948912.760603:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000010:2.0:1784948912.760609:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000090e76bb9. 00000400:00000200:2.0:1784948912.760615:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:2.0:1784948912.760623:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:2.0:1784948912.760630:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.760636:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.760642:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.760648:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:2.0:1784948912.760655:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3384:3384:244:4294967295] 111.222.173.8@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:2.0:1784948912.760664:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:2.0:1784948912.760674:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.760713:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784948912.760719:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000353a5c12. 00000800:00000200:2.0:1784948912.760728:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784948912.760736:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:2.0:1784948912.760762:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:2.0:1784948912.760767:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:2.0:1784948912.760775:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:2.0:1784948912.760784:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007db69e48. 00000400:00000010:2.0:1784948912.760791:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000400:00000010:2.0:1784948912.760801:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ead52bd9. 00000400:00000200:2.0:1784948912.760808:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:2.0:1784948912.760822:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:2.0:1784948912.760828:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.760836:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.760841:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.760847:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:2.0:1784948912.760854:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3385:3385:243:4294967295] 111.222.142.8@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:2.0:1784948912.760863:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:2.0:1784948912.760876:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.760885:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784948912.760892:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000635133da. 00000800:00000200:2.0:1784948912.760901:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.0:1784948912.760973:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 14 secs later) 00000800:00000200:2.1:1784948912.760983:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784948912.760987:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 14 secs later) 00000800:00000100:2.0:1784948912.760994:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784948912.761000:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000635133da. 00000400:00000200:2.0:1784948912.761007:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948912.761013:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948912.761018:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948912.761022:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:2.0:1784948912.761029:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:2.0:1784948912.761042:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:2.0:1784948912.761046:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 rc 0 00000400:00000200:2.0:1784948912.761054:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:2.0:1784948912.761067:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000075318cf7. 00000400:00000010:2.0:1784948912.761073:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000400:00000010:2.0:1784948912.761079:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001da9c7e9. 00000400:00000200:2.0:1784948912.761085:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:2.0:1784948912.761094:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:2.0:1784948912.761099:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.761103:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.761107:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.761112:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:2.0:1784948912.761117:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3386:3386:243:4294967295] 111.222.173.9@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:2.0:1784948912.761124:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:2.0:1784948912.761134:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.761141:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:2.0:1784948912.761145:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000635133da. 00000800:00000200:2.0:1784948912.761153:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784948912.761160:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000400:00000200:2.0:1784948912.761166:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:2.0:1784948912.761170:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x261 rc 0 00000400:00000200:2.0:1784948912.761177:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:2.0:1784948912.761184:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e7299d92. 00000400:00000010:2.0:1784948912.761189:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000010:2.0:1784948912.761196:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a1e94473. 00000400:00000200:2.0:1784948912.761201:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:2.0:1784948912.761209:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:2.0:1784948912.761214:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.761218:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.761222:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.761227:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:2.0:1784948912.761231:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3387:3387:242:4294967295] 111.222.142.9@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:2.0:1784948912.761238:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:2.0:1784948912.761248:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.761255:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:2.0:1784948912.761260:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003ed79484. 00000800:00000200:2.0:1784948912.761267:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784948912.761273:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:2.0:1784948912.761279:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:2.0:1784948912.761283:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:2.0:1784948912.761290:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:2.0:1784948912.761297:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef171f31. 00000400:00000010:2.0:1784948912.761301:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000010:2.0:1784948912.761309:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000816e8b2e. 00000400:00000200:2.0:1784948912.761314:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:2.0:1784948912.761321:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:2.0:1784948912.761326:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.761329:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.761333:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.761337:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:2.0:1784948912.761342:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3388:3388:241:4294967295] 111.222.173.10@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:2.0:1784948912.761349:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:2.0:1784948912.761358:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.761367:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784948912.761372:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000662152ce. 00000800:00000200:2.0:1784948912.761379:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784948912.761386:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:2.0:1784948912.761413:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:2.0:1784948912.761418:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:2.0:1784948912.761425:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:2.0:1784948912.761434:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f0621080. 00000400:00000010:2.0:1784948912.761439:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000400:00000010:2.0:1784948912.761446:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000055bdc3fa. 00000400:00000200:2.0:1784948912.761452:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:2.0:1784948912.761461:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:2.0:1784948912.761481:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.761485:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.761490:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.761495:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:2.0:1784948912.761503:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3389:3389:240:4294967295] 111.222.142.10@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:2.0:1784948912.761512:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:2.0:1784948912.761544:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.761552:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:2.0:1784948912.761560:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000820f02b3. 00000800:00000200:2.0:1784948912.761573:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.0:1784948912.761578:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 14 secs later) 00000800:00000200:2.1:1784948912.761588:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784948912.761594:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 14 secs later) 00000800:00000100:2.0:1784948912.761601:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.10@tcp 00000800:00000010:2.0:1784948912.761605:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000820f02b3. 00000400:00000200:2.0:1784948912.761610:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948912.761617:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948912.761622:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948912.761627:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:2.0:1784948912.761632:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:2.0:1784948912.761643:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:2.0:1784948912.761650:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 rc 0 00000400:00000200:2.0:1784948912.761657:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:2.0:1784948912.761667:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001c9428b0. 00000400:00000010:2.0:1784948912.761671:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000400:00000010:2.0:1784948912.761696:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a0ec5307. 00000400:00000200:2.0:1784948912.761704:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:2.0:1784948912.761711:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:2.0:1784948912.761716:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.761719:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.761722:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.761725:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:2.0:1784948912.761729:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3390:3390:240:4294967295] 111.222.173.11@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:2.0:1784948912.761733:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:2.0:1784948912.761739:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.761747:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784948912.761752:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000820f02b3. 00000800:00000200:2.0:1784948912.761759:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784948912.761766:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:2.0:1784948912.761772:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:2.0:1784948912.761776:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:2.0:1784948912.761782:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:2.0:1784948912.761788:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000032fe57fd. 00000400:00000010:2.0:1784948912.761793:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000400:00000010:2.0:1784948912.761800:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000099ed7e50. 00000400:00000200:2.0:1784948912.761806:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:2.0:1784948912.761814:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:2.0:1784948912.761819:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.761823:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.761829:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.761834:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:2.0:1784948912.761841:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3391:3391:239:4294967295] 111.222.142.11@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1784948912.761849:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:2.0:1784948912.761858:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.761865:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:2.0:1784948912.761869:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e6aac0b. 00000800:00000200:2.0:1784948912.761874:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784948912.761878:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:2.0:1784948912.761882:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:2.0:1784948912.761884:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:2.0:1784948912.761888:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:2.0:1784948912.761894:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000859865aa. 00000400:00000010:2.0:1784948912.761896:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000400:00000010:2.0:1784948912.761900:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000168b2dd4. 00000400:00000200:2.0:1784948912.761932:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:2.0:1784948912.761945:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:2.0:1784948912.761950:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.761953:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.761956:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.761959:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:2.0:1784948912.761962:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3392:3392:238:4294967295] 111.222.173.12@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:2.0:1784948912.761967:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:2.0:1784948912.761973:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.761977:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784948912.761981:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b4c31151. 00000800:00000200:2.0:1784948912.761984:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784948912.761988:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:2.0:1784948912.762004:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:2.0:1784948912.762006:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:2.0:1784948912.762010:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:2.0:1784948912.762017:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:2.0:1784948912.762020:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000400:00000010:2.0:1784948912.762024:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bde5615f. 00000400:00000200:2.0:1784948912.762027:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:2.0:1784948912.762032:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:2.0:1784948912.762034:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.762037:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762040:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.762042:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:2.0:1784948912.762045:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3393:3393:237:4294967295] 111.222.142.12@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:2.0:1784948912.762049:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:2.0:1784948912.762054:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762058:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784948912.762062:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a12a46b5. 00000800:00000200:2.0:1784948912.762067:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.0:1784948912.762071:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 14 secs later) 00000800:00000200:2.1:1784948912.762076:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784948912.762079:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 14 secs later) 00000800:00000100:2.0:1784948912.762086:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784948912.762089:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a12a46b5. 00000400:00000200:2.0:1784948912.762100:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948912.762104:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948912.762109:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948912.762112:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:2.0:1784948912.762116:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:2.0:1784948912.762122:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:2.0:1784948912.762125:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 rc 0 00000400:00000200:2.0:1784948912.762129:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:2.0:1784948912.762132:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.762134:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.762136:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:2.0:1784948912.762139:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:2.0:1784948912.762142:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.762145:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:2.0:1784948912.762147:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 00000400:00000200:2.0:1784948912.762151:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-5 00000400:00000200:2.0:1784948912.762153:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -5 00000400:00000200:2.0:1784948912.762156:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:2.0:1784948912.762159:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:2.0:1784948912.762164:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000015713dee. 00000400:00000010:2.0:1784948912.762166:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000010:2.0:1784948912.762169:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000007473943. 00000400:00000200:2.0:1784948912.762172:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:2.0:1784948912.762176:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:2.0:1784948912.762179:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.762182:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762184:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.762187:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:2.0:1784948912.762191:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3394:3394:237:4294967295] 111.222.173.13@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:2.0:1784948912.762197:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:2.0:1784948912.762204:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762213:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:2.0:1784948912.762215:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a12a46b5. 00000800:00000200:2.0:1784948912.762219:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784948912.762224:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:2.0:1784948912.762229:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:2.0:1784948912.762232:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:2.0:1784948912.762235:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:2.0:1784948912.762239:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000070ada26c. 00000400:00000010:2.0:1784948912.762243:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000400:00000010:2.0:1784948912.762256:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ce76e0d8. 00000400:00000200:2.0:1784948912.762260:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:2.0:1784948912.762264:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:2.0:1784948912.762268:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.762272:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762274:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.762277:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:2.0:1784948912.762280:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3395:3395:236:4294967295] 111.222.142.13@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:2.0:1784948912.762285:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:2.0:1784948912.762290:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762294:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:2.0:1784948912.762297:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e5f52141. 00000800:00000200:2.0:1784948912.762300:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784948912.762303:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:2.0:1784948912.762307:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:2.0:1784948912.762308:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:2.0:1784948912.762311:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:2.0:1784948912.762315:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 00000400:00000010:2.0:1784948912.762317:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000400:00000010:2.0:1784948912.762321:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000034b6aef4. 00000400:00000200:2.0:1784948912.762325:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:2.0:1784948912.762329:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:2.0:1784948912.762332:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.762335:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762341:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.762344:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:2.0:1784948912.762350:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3396:3396:235:4294967295] 111.222.173.14@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:2.0:1784948912.762355:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:2.0:1784948912.762372:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762376:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784948912.762380:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006c136176. 00000800:00000200:2.0:1784948912.762384:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784948912.762390:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:2.0:1784948912.762396:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:2.0:1784948912.762399:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:2.0:1784948912.762402:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:2.0:1784948912.762408:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006cd4526f. 00000400:00000010:2.0:1784948912.762410:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000400:00000010:2.0:1784948912.762413:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002a00e572. 00000400:00000200:2.0:1784948912.762416:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:2.0:1784948912.762420:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:2.0:1784948912.762425:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.762428:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762431:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.762439:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:2.0:1784948912.762447:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3397:3397:234:4294967295] 111.222.142.14@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:2.0:1784948912.762452:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:2.0:1784948912.762457:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762461:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:2.0:1784948912.762464:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000042e55fb6. 00000800:00000200:2.0:1784948912.762467:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.0:1784948912.762469:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 14 secs later) 00000800:00000200:2.1:1784948912.762474:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.1:1784948912.762477:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 14 secs later) 00000800:00000100:2.0:1784948912.762481:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.14@tcp 00000800:00000010:2.0:1784948912.762483:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000042e55fb6. 00000400:00000200:2.0:1784948912.762487:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948912.762491:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948912.762494:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948912.762496:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:2.0:1784948912.762500:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:2.0:1784948912.762505:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:2.0:1784948912.762507:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 rc 0 00000400:00000200:2.0:1784948912.762512:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:2.0:1784948912.762518:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001df3b40e. 00000400:00000010:2.0:1784948912.762520:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000400:00000010:2.0:1784948912.762525:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000068539c1f. 00000400:00000200:2.0:1784948912.762528:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:2.0:1784948912.762533:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:2.0:1784948912.762536:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.762538:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762541:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.762545:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:2.0:1784948912.762548:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3398:3398:234:4294967295] 111.222.173.15@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:2.0:1784948912.762554:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:2.0:1784948912.762562:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762567:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784948912.762569:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000042e55fb6. 00000800:00000200:2.0:1784948912.762577:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784948912.762582:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:2.0:1784948912.762602:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:2.0:1784948912.762606:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:2.0:1784948912.762612:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:2.0:1784948912.762616:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003cdea007. 00000400:00000010:2.0:1784948912.762620:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000010:2.0:1784948912.762626:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b76552f6. 00000400:00000200:2.0:1784948912.762631:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:2.0:1784948912.762644:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:2.0:1784948912.762648:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948912.762652:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762655:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948912.762657:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:2.0:1784948912.762662:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3399:3399:233:4294967295] 111.222.142.15@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:2.0:1784948912.762667:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:2.0:1784948912.762699:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762705:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:2.0:1784948912.762711:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:2.0:1784948912.762715:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784948912.762720:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:2.0:1784948912.762724:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:2.0:1784948912.762726:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:2.0:1784948912.762732:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.27@tcp(00000000898cc47a) state 0x40 00000400:00000010:2.0:1784948912.762739:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005cc3cd5d. 00000400:00000010:2.0:1784948912.762744:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000400:00000010:2.0:1784948912.762753:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001fa0e430. 00000400:00000200:2.0:1784948912.762760:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.27@tcp 00000400:00000200:2.0:1784948912.762770:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.173.27@tcp local destination 00000400:00000200:2.0:1784948912.762777:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948912.762782:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.173.27@tcp 00000400:00000200:2.0:1784948912.762789:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.173.27@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:2.0:1784948912.762796:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3400:3400:232:4294967295] 111.222.173.27@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1784948912.762804:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.27@tcp 00000400:00000200:2.0:1784948912.762817:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.27@tcp(111.222.173.27@tcp:111.222.173.27@tcp) : GET try# 0 00000800:00000200:2.0:1784948912.762827:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.27@tcp 00000800:00000010:2.0:1784948912.762834:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007a3f8609. 00000800:00000010:2.0:1784948912.762847:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000009f30905a. 00000800:00000010:2.0:1784948912.762855:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 000000009b696152. 00000800:00000200:2.0:1784948912.762864:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009f30905a] -> 12345-111.222.173.27@tcp (2) 00000800:00000200:2.1:1784948912.762870:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009f30905a] -> 12345-111.222.173.27@tcp (2) 00000400:00000200:2.0:1784948912.762895:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.27@tcp 00000400:00000200:2.0:1784948912.762899:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.27@tcp(00000000898cc47a) state 0x240 rc 0 00000400:00000200:2.0:1784948912.762943:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 00000400:00000001:2.0:1784948912.762950:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.762954:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.762957:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:2.0:1784948912.762962:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:2.0:1784948912.762969:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.762978:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:2.0:1784948912.762983:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000400:00000200:2.0:1784948912.762993:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-5 00000400:00000200:2.0:1784948912.763000:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -5 00000400:00000200:2.0:1784948912.763007:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:2.0:1784948912.763012:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:2.0:1784948912.763016:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.763018:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.763019:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:2.0:1784948912.763022:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000400:00000200:2.0:1784948912.763026:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.763031:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:2.0:1784948912.763036:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007db69e48. 00000400:00000200:2.0:1784948912.763047:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-5 00000400:00000200:2.0:1784948912.763053:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -5 00000400:00000200:2.0:1784948912.763059:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000400:00000200:2.0:1784948912.763067:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:2.0:1784948912.763071:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.763073:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.763076:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:2.0:1784948912.763078:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000400:00000200:2.0:1784948912.763082:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.763088:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:2.0:1784948912.763094:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f0621080. 00000400:00000200:2.0:1784948912.763099:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-5 00000400:00000200:2.0:1784948912.763104:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -5 00000400:00000200:2.0:1784948912.763111:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000400:00000200:2.0:1784948912.763119:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:2.0:1784948912.763123:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.763126:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.763129:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000010:2.0:1784948912.763132:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000400:00000200:2.0:1784948912.763138:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.763143:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:2.0:1784948912.763148:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000400:00000200:2.0:1784948912.763155:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-5 00000400:00000200:2.0:1784948912.763164:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -5 00000400:00000200:2.0:1784948912.763173:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000400:00000200:2.0:1784948912.763179:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:2.0:1784948912.763183:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948912.763188:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948912.763192:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:2.0:1784948912.763195:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000400:00000200:2.0:1784948912.763201:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948912.763206:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:2.0:1784948912.763211:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006cd4526f. 00000400:00000200:2.0:1784948912.763219:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-5 00000400:00000200:2.0:1784948912.763226:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -5 00000400:00000200:2.0:1784948912.763232:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00010000:00000001:2.0:1784948913.396067:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948913.396080:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948913.396090:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948913.396092:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948913.396227:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948913.396235:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948913.396241:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948913.588027:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948913.588046:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948913.588048:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948913.588054:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948913.588058:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948913.588062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948913.588064:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784948913.780088:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000800:00000010:3.0:1784948913.780096:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009bb74647. 00000400:00000200:3.0:1784948913.780109:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948913.780117:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948913.780130:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948913.780137:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:3.0:1784948913.780143:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000800:00000100:3.0:1784948913.780189:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000800:00000010:3.0:1784948913.780193:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000263f8d59. 00000400:00000200:3.0:1784948913.780197:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948913.780203:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948913.780217:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948913.780221:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:3.0:1784948913.780231:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:2.0:1784948913.780273:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784948913.780286:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:2.0:1784948913.780290:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2861 00000400:00000010:1.0:1784948913.780301:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000090a03b90. 00000400:00000001:2.0:1784948913.780304:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948913.780326:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948913.780331:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000010:2.0:1784948913.780335:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000400:00000010:1.0:1784948913.780337:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf7eadde. 00000400:00000200:2.0:1784948913.780344:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000010:1.0:1784948913.780351:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000200:2.0:1784948913.780355:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:2.0:1784948913.780366:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f1904ca. 00000400:00000200:2.0:1784948913.780376:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000010:1.0:1784948913.780377:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002e0c2c48. 00000400:00000200:2.0:1784948913.780383:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:2.0:1784948913.780397:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:1.0:1784948913.780400:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948913.780407:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2861 00000400:00000001:2.0:1784948913.780416:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948913.780419:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948913.780420:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000400:00000010:2.0:1784948913.780422:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000400:00000200:2.0:1784948913.780426:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948913.780435:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948913.780449:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948913.780458:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3401:3401:233:4294967295] 192.168.202.137@tcp LPNI seq info [2188:2188:8:4294967295] 00000400:00000200:1.0:1784948913.780473:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948913.780489:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948913.780502:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948913.780509:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:2.0:1784948913.780515:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006200396a. 00000800:00000010:1.0:1784948913.780518:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000400:00000200:2.0:1784948913.780527:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:2.0:1784948913.780531:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:2.0:1784948913.780539:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000800:00000200:1.0:1784948913.780544:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948913.780571:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948913.780581:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948913.780617:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11187 health :600 00000800:00000200:2.0:1784948913.780859:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948913.780881:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:2.0:1784948913.780890:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948913.780895:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784948913.780900:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784948913.780905:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948913.780938:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948913.780960:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948913.780966:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948913.780970:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948913.780981:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948913.781001:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784948913.781013:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784948913.781029:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948913.781037:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3402:3402:233:4294967295] 192.168.202.137@tcp LPNI seq info [2189:2189:8:4294967295] 00000400:00000200:3.0:1784948913.781052:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784948913.781059:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948913.781065:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784948913.781074:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948913.781083:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948913.781088:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948913.781151:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784948913.781262:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948913.781268:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:3.0:1784948913.781273:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948913.781279:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948913.781296:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948913.781298:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948913.781304:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948913.781309:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948913.781315:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948913.781324:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948913.781337:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x87d9 00000800:00000001:3.0:1784948913.781346:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948913.781363:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948913.781369:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948913.781376:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948913.781384:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:3.0:1784948913.781389:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000400:00000200:3.0:1784948913.781395:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948913.781401:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000090a03b90. 00000400:00000010:3.0:1784948913.781407:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf7eadde. 00000800:00000001:3.0:1784948913.781418:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948913.781423:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948913.999583:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948913.999610:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948913.999618:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948913.999621:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948914.420054:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948914.420065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948914.420074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948914.420076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948914.420143:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948914.420149:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948914.420153:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948914.612050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948914.612065:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948914.612067:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948914.612072:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948914.612077:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948914.612079:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948914.612082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948914.612088:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948914.612090:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948914.612103:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000002cc70ea. 00000020:00000040:1.0:1784948914.612112:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948914.612119:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948914.612123:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948914.612132:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948914.612144:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000adf08f9b. 00000100:00000001:1.0:1784948914.612154:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948914.612169:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948914.612185:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948914.612189:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948914.612196:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000002cc70ea] to pc [ptlrpcd_00_03+3] req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948914.612229:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948914.612232:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948914.612237:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948914.612241:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948914.612247:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948914.612251:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000299dc118. 00000020:00000040:1.0:1784948914.612255:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948914.612256:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948914.612260:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1784948914.612262:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 02000000:00000001:1.0:1784948914.612263:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948914.612266:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948914.612268:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948914.612272:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:1.0:1784948914.612273:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000031deb17e. 00000100:00000001:2.0:1784948914.612276:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948914.612277:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948914.612280:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948914.612281:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948914.612287:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948914.612292:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948914.612294:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948914.612301:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784948914.612302:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948914.612340:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948914.612341:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948914.612343:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948914.612345:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948914.612349:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000299dc118] to pc [ptlrpcd_00_00+0] req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948914.612350:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948914.612355:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948914.612362:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948914.612366:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000002cc70ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289661312:0@lo:400: 00000100:00000001:2.0:1784948914.612371:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948914.612377:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948914.612380:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948914.612382:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948914.612383:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784948914.612390:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11191) 02000000:00000010:2.0:1784948914.612392:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 02000000:00000001:2.0:1784948914.612395:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948914.612404:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000037df9850. 00000400:00000010:2.0:1784948914.612415:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000100:00000200:2.0:1784948914.612424:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289661312, portal 25 00000100:00000001:0.0:1784948914.612426:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948914.612428:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948914.612431:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549163392 : -115544160388224 : ffff96e9c75d3b80) 00000100:00100000:0.0:1784948914.612435:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1784948914.612440:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948914.612441:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948914.612459:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948914.612461:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948914.612465:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948914.612466:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948914.612468:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948914.612479:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000100:00000001:0.0:1784948914.612480:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948914.612483:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.612492:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:2.0:1784948914.612492:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289661312, offset 0 00000100:00000001:0.0:1784948914.612501:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784948914.612509:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:0.0:1784948914.612518:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948914.612539:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948914.612556:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948914.612560:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948914.612569:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6437f80 02000000:00000001:0.0:1784948914.612571:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948914.612578:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784948914.612582:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7dcd [2] + 0 02000000:00000001:0.0:1784948914.612587:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948914.612590:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ba749c18. 00000100:00100000:0.0:1784948914.612597:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289661440:192.168.202.137@tcp:400: 00000400:00000200:2.0:1784948914.612601:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b0e9c277 00000100:00000001:0.0:1784948914.612604:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948914.612610:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1784948914.612614:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:0.0:1784948914.612615:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948914.612632:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784948914.612640:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948914.612646:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0e9c277 02000000:00000010:0.0:1784948914.612647:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f11303f8. 02000000:00000001:0.0:1784948914.612659:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948914.612687:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b0e9c277. 00000400:00000010:0.0:1784948914.612694:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000681ff0f0. 00000100:00000001:2.0:1784948914.612696:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948914.612700:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:0.0:1784948914.612705:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002e7a7cb2. 00000100:00000040:2.0:1784948914.612705:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000092af50af x1871651289661312 msgsize 224 00000100:00100000:2.0:1784948914.612722:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000200:0.0:1784948914.612724:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289661440, portal 10 00000100:00000040:2.0:1784948914.612725:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784948914.612736:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948914.612741:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549152640 : -115544160398976 : ffff96e9c75d1180) 00000100:00000001:2.0:1784948914.612750:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784948914.612756:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948914.612764:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000010:2.0:1784948914.612773:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000100:00000001:0.0:1784948914.612776:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948914.612779:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784948914.612787:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:2.0:1784948914.612788:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948914.612792:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000100:00000200:0.0:1784948914.612804:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289661440, offset 0 00000100:00000001:3.0:1784948914.612810:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948914.612814:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1784948914.612817:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289661312 02000000:00000001:3.0:1784948914.612827:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:0.0:1784948914.612831:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784948914.612831:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948914.612834:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948914.612844:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948914.612860:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948914.612864:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948914.612866:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00100000:3.0:1784948914.612871:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289661312 00000100:00000001:2.0:1784948914.612876:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784948914.612881:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948914.612889:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:0.0:1784948914.612894:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784948914.612896:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948914.612901:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:0.0:1784948914.612903:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:3.0:1784948914.612955:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784948914.612960:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1784948914.612964:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784948914.612969:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784948914.612975:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784948914.612982:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3403:3403:233:4294967295] 192.168.202.137@tcp LPNI seq info [2190:2190:8:4294967295] 00000020:00000010:3.0:1784948914.612987:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:3.0:1784948914.612994:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948914.613004:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000400:00000200:0.0:1784948914.613011:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:3.0:1784948914.613019:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000800:00000200:0.0:1784948914.613024:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948914.613024:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948914.613029:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000fe2862da. 00000100:00000001:2.0:1784948914.613048:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305853995 00000020:00000010:3.0:1784948914.613050:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003c019a34. 00000800:00000010:0.0:1784948914.613051:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b51ad697. 00000100:00000010:2.0:1784948914.613065:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fbfea82a. 00000100:00000040:3.0:1784948914.613072:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000200:0.0:1784948914.613076:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948914.613079:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948914.613080:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784948914.613092:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948914.613098:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305853995 00000800:00000200:0.0:1784948914.613099:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784948914.613100:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948914.613117:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000addba683. 00000100:00000001:3.0:1784948914.613117:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948914.613120:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b51ad697 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948914.613140:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948914.613145:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305853995 00000100:00000010:2.0:1784948914.613149:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000094130822. 00000100:00000001:2.0:1784948914.613155:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948914.613156:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948914.613156:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948914.613160:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305853995 00000100:00000040:0.0:1784948914.613167:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2093 usecs (3 jiffies) 00000100:00000010:2.0:1784948914.613176:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b9b6859c. 00000100:00000001:0.0:1784948914.613178:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305853956 00000100:00000001:0.0:1784948914.613189:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305853995 00000100:00000001:2.0:1784948914.613191:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948914.613197:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948914.613205:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948914.613212:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305853956 00000100:00000001:1.0:1784948914.613214:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948914.613215:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948914.613217:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948914.613231:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.613232:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305853995 00000100:00000001:2.0:1784948914.613235:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948914.613241:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 733 00000100:00000001:0.0:1784948914.613253:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305853956 00000100:00000001:2.0:1784948914.613255:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948914.613256:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948914.613257:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.613263:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305853995 00000100:00000001:1.0:1784948914.613274:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742038608 : -115538967513008 : ffff96eafce23850) 00000100:00000001:3.0:1784948914.613276:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.613277:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305853956 00000100:00000001:0.0:1784948914.613295:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305853995 00000100:00000040:1.0:1784948914.613303:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000092af50af x1871651289661312/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:575/0 lens 224/0 e 0 to 0 dl 1784948925 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948914.613305:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948914.613321:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.613339:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948914.613340:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948914.613350:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948914.613351:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f7efedb6. 00000100:00000001:1.0:1784948914.613360:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948914.613363:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f7efedb6. 00000100:00100000:1.0:1784948914.613372:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000092af50af pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289661312:12345-0@lo:400: 00000100:00000200:1.0:1784948914.613376:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289661312 00000020:00000001:1.0:1784948914.613380:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948914.613386:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948914.613388:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948914.613391:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948914.613394:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948914.613398:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948914.613400:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948914.613403:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948914.613406:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948914.613409:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948914.613411:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000200:2.0:1784948914.613414:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948914.613415:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948914.613419:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000010:2.0:1784948914.613421:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b51ad697. 02000000:00000010:1.0:1784948914.613425:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007f0bf97c. 00000400:00000200:2.0:1784948914.613427:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784948914.613430:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948914.613435:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948914.613436:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784948914.613438:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948914.613441:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00010000:00000040:1.0:1784948914.613443:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289661312 00000400:00000010:2.0:1784948914.613444:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000100:00000001:2.0:1784948914.613450:0:337114:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1784948914.613451:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000200:2.0:1784948914.613459:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784948914.613461:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000092af50af x1871651289661312/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:575/0 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948914.613469:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948914.613493:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:3.0:1784948914.613494:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948914.613495:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948914.613501:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948914.613504:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948914.613508:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:1.0:1784948914.613512:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=86 v=5 (1 1 1 1) 00000800:00000001:3.0:1784948914.613517:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948914.613532:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948914.613537:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948914.613540:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948914.613545:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948914.613552:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:3.0:1784948914.613556:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:1.0:1784948914.613561:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948914.613565:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948914.613569:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948914.613572:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948914.613576:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000200:3.0:1784948914.613581:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438000 00000100:00000200:1.0:1784948914.613584:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289661312, offset 224 00000400:00000200:3.0:1784948914.613595:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 159856 00000400:00000200:1.0:1784948914.613605:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:3.0:1784948914.613606:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948914.613617:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948914.613625:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948914.613627:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948914.613639:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948914.613644:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6437f80 00000100:00000001:3.0:1784948914.613658:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:1.0:1784948914.613676:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x87dd [1] + 224 00000400:00000200:1.0:1784948914.613684:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948914.613687:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948914.613691:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948914.613694:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:1.0:1784948914.613698:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948914.613708:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005dccb977. 00000100:00000040:3.0:1784948914.613720:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000005dccb977 x1871651289661440 msgsize 224 00000100:00100000:3.0:1784948914.613750:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:1.0:1784948914.613752:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948914.613774:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948914.613775:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948914.613789:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784948914.613793:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:1.0:1784948914.613797:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948914.613799:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948914.613800:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948914.613805:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948914.613806:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948914.613811:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:3.0:1784948914.613811:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948914.613813:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:2.0:1784948914.613825:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000001:1.0:1784948914.613829:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:2.0:1784948914.613831:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784948914.613834:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948914.613835:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000037df9850. 00000020:00000001:1.0:1784948914.613839:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:2.0:1784948914.613843:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000100:00000040:1.0:1784948914.613844:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000092af50af x1871651289661312/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:575/0 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948914.613848:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000100:00000001:2.0:1784948914.613853:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784948914.613862:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000092af50af pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289661312:12345-0@lo:400: Request processed in 518us (1151us total) trans 0 rc 0/0 00000100:00000200:2.0:1784948914.613864:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948914.613880:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 733 00000100:00000040:1.0:1784948914.613887:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948914.613892:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948914.613892:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:2.0:1784948914.613892:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948914.613932:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948914.613940:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000fe2862da. 00000100:00100000:0.0:1784948914.613943:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289661440 00000020:00000010:1.0:1784948914.613948:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003c019a34. 02000000:00000001:0.0:1784948914.613951:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000010:1.0:1784948914.613960:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 00000100:00000001:0.0:1784948914.613961:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948914.613967:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948914.613970:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948914.613973:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948914.613978:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948914.613979:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948914.613980:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:2.0:1784948914.613985:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948914.613995:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:0.0:1784948914.613998:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289661440 02000000:00000001:1.0:1784948914.614000:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948914.614000:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948914.614011:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007f0bf97c. 02000000:00000001:2.0:1784948914.614016:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948914.614018:0:342764:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784948914.614020:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:2.0:1784948914.614025:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=86 v=5 (5 5 5 5) 00000020:00000040:0.0:1784948914.614035:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:1.0:1784948914.614035:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784948914.614037:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=86 v=5 (1 1 1 1) 00000020:00000001:0.0:1784948914.614055:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1784948914.614058:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948914.614058:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000040:0.0:1784948914.614068:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784948914.614068:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948914.614069:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000034b95863. 00000400:00000010:1.0:1784948914.614085:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000095fae736. 00010000:00000001:2.0:1784948914.614085:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:0.0:1784948914.614087:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:2.0:1784948914.614095:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948914.614105:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948914.614106:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.614128:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784948914.614129:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948914.614144:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948914.614145:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:0.0:1784948914.614162:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948914.614162:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948914.614168:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948914.614171:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00000040:2.0:1784948914.614178:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948914.614186:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000043c3e1d8. 00000100:00100000:2.0:1784948914.614188:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000002cc70ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289661312:0@lo:400: 00000100:00000001:2.0:1784948914.614202:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948914.614204:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bcdae09a. 00000100:00000001:2.0:1784948914.614220:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1784948914.614223:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:2.0:1784948914.614227:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000002cc70ea x1871651289661312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948914.614235:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948914.614235:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948914.614238:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1784948914.614245:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000010:2.0:1784948914.614246:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 00000100:00000001:0.0:1784948914.614249:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948914.614255:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948914.614256:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948914.614269:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948914.614272:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948914.614273:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000020:00000001:2.0:1784948914.614285:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:0.0:1784948914.614286:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948914.614297:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948914.614297:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000adf08f9b. 02000000:00000001:2.0:1784948914.614319:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948914.614322:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948914.614324:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000002cc70ea. 00000100:00000001:2.0:1784948914.614329:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948914.614329:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948914.614331:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948914.614334:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948914.614335:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.614338:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948914.614338:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948914.614347:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.614370:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948914.614375:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948914.614389:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:0.0:1784948914.614390:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 709 00000100:00000001:1.0:1784948914.614401:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784948914.614406:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948914.614406:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.614419:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529482519808 : -115544227031808 : ffff96e9c3645500) 00000100:00000001:1.0:1784948914.614419:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948914.614448:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005dccb977 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:575/0 lens 224/0 e 0 to 0 dl 1784948925 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948914.614464:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948914.614468:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:0.0:1784948914.614470:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948914.614473:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00100000:0.0:1784948914.614479:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005dccb977 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289661440:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948914.614484:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289661440 00000020:00000001:0.0:1784948914.614487:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948914.614493:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948914.614496:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948914.614499:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948914.614501:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948914.614506:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948914.614510:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948914.614514:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948914.614516:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948914.614520:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948914.614522:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948914.614528:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948914.614531:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948914.614535:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fcf34dfd. 02000000:00000001:0.0:1784948914.614538:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.614541:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948914.614544:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948914.614547:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289661440 00010000:00000001:0.0:1784948914.614551:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948914.614555:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005dccb977 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:575/0 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948914.614566:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948914.614570:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948914.614574:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=81 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948914.614580:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948914.614583:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948914.614586:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948914.614589:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948914.614592:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948914.614595:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948914.614598:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948914.614602:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000100:00000200:0.0:1784948914.614607:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289661440, offset 224 00000400:00000200:0.0:1784948914.614613:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948914.614624:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948914.614632:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3404:3404:233:4294967295] 192.168.202.137@tcp LPNI seq info [2191:2191:8:4294967295] 00000400:00000200:0.0:1784948914.614644:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948914.614650:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948914.614654:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000eafb2cdc. 00000800:00000200:0.0:1784948914.614659:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948914.614696:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948914.614701:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000eafb2cdc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948914.614720:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948914.614724:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948914.614726:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948914.614728:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948914.614731:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948914.614735:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005dccb977 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:575/0 lens 224/224 e 0 to 0 dl 1784948925 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948914.614746:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005dccb977 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289661440:12345-192.168.202.137@tcp:400: Request processed in 282us (1025us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948914.614755:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 709 00000100:00000040:0.0:1784948914.614759:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948914.614762:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948914.614764:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948914.614770:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bcdae09a. 00000020:00000010:0.0:1784948914.614777:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948914.614783:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000043c3e1d8. 00000020:00000040:0.0:1784948914.614790:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948914.614794:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948914.614875:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948914.614879:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000eafb2cdc. 00000400:00000200:2.0:1784948914.614883:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948914.614889:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948914.614893:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:2.0:1784948914.614895:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000100:00000001:2.0:1784948914.614898:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948914.614900:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948914.614934:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948914.614948:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948914.614952:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948914.614960:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948914.614967:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948914.614980:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438000 00000400:00000200:0.0:1784948914.614989:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x87e5 [1] + 224 00000800:00000001:0.0:1784948914.614996:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948914.615005:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948914.615009:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948914.615013:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948914.615020:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948914.615024:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948914.615037:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948914.615060:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948914.615068:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948914.615072:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948914.615103:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948914.615108:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948914.615113:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948914.615115:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948914.615117:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000681ff0f0. 00000400:00000200:1.0:1784948914.615122:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e7a7cb2 00000400:00000010:1.0:1784948914.615124:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002e7a7cb2. 00000100:00000001:1.0:1784948914.615127:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948914.615133:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948914.615141:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948914.615148:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948914.615149:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948914.615151:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948914.615152:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948914.615154:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948914.615155:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948914.615157:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948914.615162:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=81 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948914.615165:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=81 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948914.615167:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948914.615168:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948914.615169:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948914.615172:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948914.615177:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948914.615180:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948914.615181:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948914.615182:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948914.615183:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948914.615185:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948914.615191:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948914.615193:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948914.615194:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948914.615196:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948914.615201:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000299dc118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289661440:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948914.615204:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948914.615205:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948914.615207:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000299dc118 x1871651289661440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948914.615211:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948914.615212:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948914.615213:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f11303f8. 02000000:00000001:1.0:1784948914.615216:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948914.615217:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948914.615218:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948914.615220:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948914.615221:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000031deb17e. 02000000:00000001:1.0:1784948914.615223:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948914.615224:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948914.615225:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000299dc118. 00000100:00000001:1.0:1784948914.615227:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948914.615228:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948914.615229:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948914.615233:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948914.615233:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948914.615235:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948914.615237:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948914.656016:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305853995, at 4305853996 00000100:00000040:1.0:1784948914.656026:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305853995, at 4305853996 00000100:00000010:2.0:1784948914.656030:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000094130822. 00000100:00000010:1.0:1784948914.656040:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000addba683. 00000100:00000040:3.0:1784948914.656042:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305853995, at 4305853996 00000100:00000010:3.0:1784948914.656062:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b9b6859c. 00000100:00000040:0.0:1784948914.656065:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305853995, at 4305853996 00000100:00000010:0.0:1784948914.656080:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fbfea82a. 00010000:00000001:2.0:1784948915.444081:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948915.444094:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948915.444104:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948915.444105:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948915.444215:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948915.444225:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948915.444230:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948915.636142:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948915.636159:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948915.636163:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948915.636170:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948915.636174:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948915.636178:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948915.636182:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948916.468088:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948916.468105:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948916.468116:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948916.468118:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948916.468249:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948916.468257:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948916.468262:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948916.660083:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948916.660108:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948916.660111:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948916.660119:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948916.660124:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948916.660130:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948916.660135:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948916.852021:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000022dbcdce. 00000400:00000010:1.0:1784948916.852053:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000851e368. 00000400:00000010:1.0:1784948916.852066:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009f8441cb. 00000400:00000010:1.0:1784948916.852078:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e3180272. 00000400:00000010:1.0:1784948916.852089:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d3823a3e. 00000400:00000010:1.0:1784948916.852100:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f9fb66e4. 00000400:00000010:1.0:1784948916.852104:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000007e2f3d0. 00000400:00000010:1.0:1784948916.852110:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a3f431e3. 00000400:00000010:1.0:1784948916.852113:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000014336a14. 00000400:00000010:1.0:1784948916.852117:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e3882e96. 00000400:00000010:1.0:1784948916.852120:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009ec02d9a. 00000400:00000010:1.0:1784948916.852122:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cef8b3cd. 00000400:00000010:1.0:1784948916.852124:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a8bf092e. 00000400:00000010:1.0:1784948916.852126:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000339be500. 00000400:00000010:1.0:1784948916.852127:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000171e706a. 00000400:00000010:1.0:1784948916.852129:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006d2c4b24. 00000400:00000010:1.0:1784948916.852130:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000091a157d3. 00000400:00000010:1.0:1784948916.852132:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003b463cff. 00000400:00000010:1.0:1784948916.852134:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f65f2fb7. 00000400:00000010:1.0:1784948916.852135:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000041ec6752. 00000400:00000010:1.0:1784948916.852137:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009e0071c7. 00000400:00000010:1.0:1784948916.852138:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f12dd04d. 00000400:00000010:1.0:1784948916.852140:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003faefc6b. 00000400:00000010:1.0:1784948916.852141:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000afa63392. 00000400:00000010:1.0:1784948916.852143:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003859b1c5. 00000400:00000010:1.0:1784948916.852144:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000548ea7b. 00000400:00000010:1.0:1784948916.852146:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c241e266. 00000400:00000010:1.0:1784948916.852148:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a705132d. 00000400:00000010:1.0:1784948916.852149:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000758fd515. 00000400:00000010:1.0:1784948916.852151:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000003b6cd72. 00000400:00000010:1.0:1784948916.852153:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008015f6b2. 00000400:00000010:1.0:1784948916.852154:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cf871ac3. 00000400:00000010:1.0:1784948916.852156:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009234a5fd. 00000400:00000010:1.0:1784948916.852157:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000451e3fe4. 00000400:00000010:1.0:1784948916.852160:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000acbf6818. 00000400:00000010:1.0:1784948916.852163:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000056670828. 00000400:00000010:1.0:1784948916.852170:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000a5d96d2. 00000400:00000010:1.0:1784948916.852171:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000063d06960. 00000400:00000010:1.0:1784948916.852173:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006b55cc75. 00000400:00000010:1.0:1784948916.852175:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c3236cd0. 00000400:00000010:1.0:1784948916.852176:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000735cb99e. 00000400:00000010:1.0:1784948916.852178:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000005547c433. 00000400:00000010:1.0:1784948916.852179:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009e875900. 00000400:00000010:1.0:1784948916.852181:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000038aa1d2e. 00000400:00000010:1.0:1784948916.852183:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000147b89d3. 00000400:00000010:1.0:1784948916.852184:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000067f2d25d. 00000400:00000010:1.0:1784948916.852186:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000dceddabe. 00000400:00000010:1.0:1784948916.852188:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000756d087e. 00000400:00000010:1.0:1784948916.852189:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000096383a5. 00000400:00000010:1.0:1784948916.852191:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000004d10c17. 00000400:00000010:1.0:1784948916.852192:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003c8b2437. 00000400:00000010:1.0:1784948916.852194:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a7ce911a. 00000400:00000010:1.0:1784948916.852195:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003d75c981. 00000400:00000010:1.0:1784948916.852197:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000383d6bd1. 00000400:00000010:1.0:1784948916.852198:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001f662712. 00000400:00000010:1.0:1784948916.852200:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bfef55e5. 00000400:00000010:1.0:1784948916.852201:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000054872d18. 00000400:00000010:1.0:1784948916.852203:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b03d8085. 00000400:00000010:1.0:1784948916.852204:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000934e2586. 00000400:00000010:1.0:1784948916.852206:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d4c102f3. 00000400:00000010:1.0:1784948916.852207:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001fcbecf4. 00000400:00000010:1.0:1784948916.852209:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008c245eeb. 00000400:00000010:1.0:1784948916.852211:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000038468d93. 00000400:00000010:1.0:1784948916.852212:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bd6a7269. 00010000:00000001:2.0:1784948917.492073:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948917.492088:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948917.492097:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948917.492100:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948917.492215:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948917.492224:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948917.492230:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948917.684113:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948917.684130:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948917.684132:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948917.684138:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948917.684142:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948917.684147:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948917.684149:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948918.515999:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948918.516011:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948918.516018:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948918.516020:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948918.516099:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948918.516104:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948918.516108:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948918.708068:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948918.708085:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948918.708088:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948918.708093:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948918.708096:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948918.708101:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948918.708104:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948919.540068:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948919.540083:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948919.540097:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948919.540100:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948919.540265:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948919.540276:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948919.540281:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948919.732029:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948919.732045:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948919.732048:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948919.732054:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948919.732057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948919.732060:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948919.732064:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948919.732111:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948919.732126:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948919.732142:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b8226094. 00000020:00000040:2.0:1784948919.732151:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948919.732159:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948919.732163:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948919.732173:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948919.732185:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000cf155122. 00000100:00000001:2.0:1784948919.732196:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948919.732209:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948919.732225:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948919.732228:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948919.732235:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b8226094] to pc [ptlrpcd_00_01+1] req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948919.732273:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948919.732276:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948919.732281:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948919.732284:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948919.732291:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948919.732298:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008d623923. 00000020:00000040:2.0:1784948919.732301:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948919.732304:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948919.732306:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948919.732308:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948919.732312:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000cee99234. 00000100:00000001:2.0:1784948919.732315:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.732316:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948919.732325:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784948919.732325:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948919.732335:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948919.732345:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948919.732346:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948919.732353:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948919.732355:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d623923] to pc [ptlrpcd_00_02+2] req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948919.732370:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948919.732384:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.732471:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948919.732475:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948919.732481:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948919.732483:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948919.732485:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948919.732493:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11196) 02000000:00000001:1.0:1784948919.732494:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.732507:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948919.732515:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289661568:0@lo:400: 00000100:00000001:3.0:1784948919.732516:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948919.732524:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948919.732536:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948919.732542:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948919.732547:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784948919.732547:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948919.732550:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948919.732556:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948919.732574:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784948919.732589:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:1.0:1784948919.732592:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dbce179e. 00000100:00000001:3.0:1784948919.732593:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948919.732613:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948919.732615:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:1.0:1784948919.732649:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000789ee428. 02000000:00000001:3.0:1784948919.732653:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948919.732672:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 02000000:00000001:3.0:1784948919.732679:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948919.732685:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289661568, portal 25 00000100:00100000:3.0:1784948919.732687:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289661696:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948919.732699:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948919.732701:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784948919.732703:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470022400 : -115544239529216 : ffff96e9c2a5a300) 02000000:00000001:3.0:1784948919.732710:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948919.732714:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948919.732715:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948919.732726:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948919.732740:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:3.0:1784948919.732740:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 02000000:00000001:3.0:1784948919.732751:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948919.732757:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948919.732763:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000200:1.0:1784948919.732770:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289661568, offset 0 00000400:00000010:3.0:1784948919.732772:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000de368614. 00000400:00000200:1.0:1784948919.732791:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784948919.732793:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000100:00000200:3.0:1784948919.732818:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289661696, portal 10 00000100:00000001:3.0:1784948919.732823:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948919.732830:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529470027776 : -115544239523840 : ffff96e9c2a5b800) 00000400:00000200:1.0:1784948919.732836:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784948919.732851:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948919.732863:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438080 00000100:00000001:3.0:1784948919.732893:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784948919.732895:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7df1 [2] + 0 00000100:00000040:3.0:1784948919.732910:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948919.732914:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bb55de7f. 00000400:00000010:1.0:1784948919.732946:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000400:00000200:1.0:1784948919.732956:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000e65b0b8e 00000400:00000200:1.0:1784948919.732969:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:3.0:1784948919.732970:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289661696, offset 0 00000400:00000200:1.0:1784948919.732980:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784948919.732992:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948919.733001:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e65b0b8e 00000400:00000010:1.0:1784948919.733007:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000e65b0b8e. 00000400:00000200:3.0:1784948919.733017:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784948919.733018:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948919.733031:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1784948919.733036:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1784948919.733040:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000004a69f61a x1871651289661568 msgsize 224 00000400:00000200:3.0:1784948919.733047:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948919.733067:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00100000:1.0:1784948919.733068:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784948919.733079:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948919.733082:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948919.733093:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3405:3405:233:4294967295] 192.168.202.137@tcp LPNI seq info [2192:2192:8:4294967295] 00000100:00000001:1.0:1784948919.733107:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948919.733115:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948919.733123:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948919.733129:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000400:00000200:1.0:1784948919.733132:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:1.0:1784948919.733137:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:1.0:1784948919.733141:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784948919.733148:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:1.0:1784948919.733157:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948919.733177:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784948919.733183:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784948919.733185:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000040:1.0:1784948919.733192:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948919.733214:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.733216:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.733220:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784948919.733225:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784948919.733225:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2095 usecs (3 jiffies) 00000100:00100000:2.0:1784948919.733234:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289661568 00000100:00000001:3.0:1784948919.733236:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305859076 00000100:00000001:1.0:1784948919.733238:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948919.733242:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948919.733248:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000010:3.0:1784948919.733255:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000006204bf6f. 00000100:00000001:2.0:1784948919.733256:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.733260:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305859115 02000000:00000001:2.0:1784948919.733269:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.733271:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00100000:2.0:1784948919.733279:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289661568 00000020:00000001:2.0:1784948919.733284:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948919.733286:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305859076 00000020:00000040:2.0:1784948919.733287:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784948919.733295:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:3.0:1784948919.733299:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000100:00000001:1.0:1784948919.733300:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305859076 00000020:00000040:2.0:1784948919.733301:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948919.733316:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000001:2.0:1784948919.733318:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784948919.733327:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948919.733335:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305859115 00000100:00000001:2.0:1784948919.733335:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948919.733336:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305859076 00000020:00000010:2.0:1784948919.733347:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000001:2.0:1784948919.733359:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1784948919.733361:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000029366a5. 00000100:00000001:1.0:1784948919.733363:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305859076 00000020:00000010:2.0:1784948919.733365:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000100:00000001:3.0:1784948919.733373:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000010:2.0:1784948919.733389:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006cd4526f. 00000100:00000001:1.0:1784948919.733392:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305859115 00000100:00000001:3.0:1784948919.733394:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305859076 00000020:00000010:2.0:1784948919.733401:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c335e9c8. 00000800:00000200:0.0:1784948919.733408:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000010:3.0:1784948919.733415:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000100:00000001:1.0:1784948919.733416:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305859076 00000020:00000010:2.0:1784948919.733418:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000800:00000010:0.0:1784948919.733419:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000100:00000001:3.0:1784948919.733427:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948919.733430:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305859115 00000400:00000200:0.0:1784948919.733449:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948919.733451:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948919.733458:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948919.733459:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305859076 00000100:00000001:3.0:1784948919.733476:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.733483:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948919.733485:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948919.733491:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.733492:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784948919.733496:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948919.733498:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.733499:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.733509:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948919.733516:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bb55de7f 00000400:00000010:0.0:1784948919.733527:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bb55de7f. 00000100:00000001:1.0:1784948919.733528:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.733530:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948919.733534:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.733544:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948919.733546:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948919.733549:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948919.733557:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948919.733568:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948919.733573:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948919.733576:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948919.733581:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948919.733584:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948919.733586:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948919.733591:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.733600:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948919.733602:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948919.733608:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.733612:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948919.733645:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948919.733646:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 734 00000100:00000001:3.0:1784948919.733663:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:2.0:1784948919.733676:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948919.733680:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.733682:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948919.733696:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.733697:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742032464 : -115538967519152 : ffff96eafce22050) 00000100:00000001:0.0:1784948919.733701:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948919.733708:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948919.733722:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948919.733723:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000004a69f61a x1871651289661568/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:580/0 lens 224/0 e 0 to 0 dl 1784948930 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948919.733735:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948919.733755:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948919.733772:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948919.733783:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948919.733819:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948919.733823:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948919.733837:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948919.733840:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:2.0:1784948919.733844:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000001:2.0:1784948919.733850:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948919.733852:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000100:00100000:2.0:1784948919.733859:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000004a69f61a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289661568:12345-0@lo:400: 00000400:00000200:1.0:1784948919.733873:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438100 00000100:00000200:2.0:1784948919.733874:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289661568 00000020:00000001:2.0:1784948919.733887:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:1.0:1784948919.733894:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 160080 00000020:00000001:2.0:1784948919.733895:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948919.733903:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948919.733941:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948919.733944:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000001:1.0:1784948919.733946:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784948919.733958:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948919.733961:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948919.733965:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948919.733965:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948919.733968:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:1.0:1784948919.733975:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784948919.733976:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948919.733993:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:1.0:1784948919.734004:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948919.734005:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948919.734015:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948919.734018:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948919.734022:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000010:2.0:1784948919.734023:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000de31af8d. 00000100:00000010:1.0:1784948919.734029:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000299dc118. 02000000:00000001:2.0:1784948919.734029:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948919.734045:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000299dc118 x1871651289661696 msgsize 224 00000100:00000001:2.0:1784948919.734045:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948919.734060:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948919.734072:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289661568 00000100:00100000:1.0:1784948919.734081:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:2.0:1784948919.734083:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948919.734091:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000004a69f61a x1871651289661568/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:580/0 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948919.734099:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000001:2.0:1784948919.734103:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948919.734107:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948919.734111:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:2.0:1784948919.734114:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=91 v=5 (1 1 1 1) 00000800:00000001:1.0:1784948919.734117:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948919.734122:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948919.734128:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948919.734133:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784948919.734138:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948919.734141:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.734146:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948919.734152:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948919.734157:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000100:00000001:0.0:1784948919.734162:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:2.0:1784948919.734163:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289661568, offset 224 00000100:00100000:0.0:1784948919.734170:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289661696 02000000:00000001:0.0:1784948919.734178:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:2.0:1784948919.734178:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948919.734189:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948919.734193:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948919.734199:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948919.734200:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:0.0:1784948919.734215:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289661696 00000020:00000001:0.0:1784948919.734220:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1784948919.734227:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438080 00000020:00000040:0.0:1784948919.734228:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784948919.734243:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948919.734249:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:2.0:1784948919.734253:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x87f9 [1] + 224 00000020:00000001:0.0:1784948919.734261:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948919.734268:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784948919.734272:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948919.734274:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784948919.734284:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:0.0:1784948919.734286:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:2.0:1784948919.734295:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948919.734297:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948919.734307:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000200:2.0:1784948919.734308:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948919.734327:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000043c3e1d8. 00000100:00000040:2.0:1784948919.734333:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948919.734338:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bcdae09a. 00000020:00000010:0.0:1784948919.734352:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784948919.734362:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784948919.734366:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948919.734368:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948919.734371:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948919.734372:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948919.734383:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:2.0:1784948919.734384:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000400:00000010:2.0:1784948919.734398:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000100:00000001:0.0:1784948919.734399:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948919.734406:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.734409:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948919.734413:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948919.734420:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.734426:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948919.734426:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784948919.734431:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948919.734440:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784948919.734442:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948919.734451:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000001:2.0:1784948919.734455:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948919.734458:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000001:2.0:1784948919.734460:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:1.0:1784948919.734462:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948919.734466:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1784948919.734469:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004a69f61a x1871651289661568/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:580/0 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948919.734474:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000789ee428. 00000100:00100000:0.0:1784948919.734478:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 710 00000400:00000200:1.0:1784948919.734489:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000100:00000040:0.0:1784948919.734496:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000010:1.0:1784948919.734501:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:0.0:1784948919.734511:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549152640 : -115544160398976 : ffff96e9c75d1180) 00000100:00000001:1.0:1784948919.734521:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784948919.734535:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000299dc118 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:580/0 lens 224/0 e 0 to 0 dl 1784948930 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948919.734536:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948919.734550:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000004a69f61a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289661568:12345-0@lo:400: Request processed in 715us (1497us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948919.734616:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 734 00000100:00000040:2.0:1784948919.734646:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:1.0:1784948919.734655:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948919.734661:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948919.734662:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948919.734674:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948919.734675:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:0.0:1784948919.734685:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948919.734686:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c335e9c8. 00000020:00000010:0.0:1784948919.734701:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000020:00000010:2.0:1784948919.734704:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784948919.734720:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006cd4526f. 00000100:00000001:1.0:1784948919.734722:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:0.0:1784948919.734723:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000299dc118 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289661696:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948919.734733:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784948919.734742:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:0.0:1784948919.734745:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289661696 00000100:00000001:1.0:1784948919.734745:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948919.734752:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:0.0:1784948919.734754:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:2.0:1784948919.734754:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784948919.734757:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000010:2.0:1784948919.734760:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000de31af8d. 00000020:00000001:0.0:1784948919.734766:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784948919.734766:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948919.734769:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:0.0:1784948919.734771:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948919.734777:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948919.734778:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948919.734779:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948919.734796:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00001000:1.0:1784948919.734797:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=91 v=5 (5 5 5 5) 00000100:00000200:2.0:1784948919.734798:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:0.0:1784948919.734808:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00001000:1.0:1784948919.734812:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=91 v=5 (1 1 1 1) 00000400:00000010:2.0:1784948919.734814:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000037df9850. 00000020:00000001:0.0:1784948919.734817:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948919.734826:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000cd71eba3. 00000100:00000001:1.0:1784948919.734829:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:0.0:1784948919.734831:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948919.734841:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948919.734843:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00010000:00000001:1.0:1784948919.734846:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:0.0:1784948919.734852:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948919.734852:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.734862:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948919.734865:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:1.0:1784948919.734869:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948919.734877:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948919.734879:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948919.734885:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948919.734885:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948919.734889:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948919.734893:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784948919.734896:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c6596e24. 02000000:00000001:0.0:1784948919.734905:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948919.734938:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b8226094 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289661568:0@lo:400: 00000100:00000001:1.0:1784948919.734947:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948919.734950:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.734952:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948919.734959:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b8226094 x1871651289661568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948919.734961:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948919.734974:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289661696 00000100:00000001:1.0:1784948919.734976:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948919.734982:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00010000:00000001:0.0:1784948919.734984:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000010:1.0:1784948919.734986:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dbce179e. 02000000:00000001:1.0:1784948919.734996:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00010000:00000200:0.0:1784948919.734999:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000299dc118 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:580/0 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948919.735001:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948919.735004:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784948919.735010:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948919.735014:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000cf155122. 02000000:00000001:1.0:1784948919.735022:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948919.735027:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948919.735032:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b8226094. 00010000:00000001:0.0:1784948919.735035:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784948919.735037:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948919.735041:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948919.735042:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.735044:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948919.735055:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00001000:0.0:1784948919.735056:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=86 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948919.735058:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948919.735066:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948919.735071:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948919.735073:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948919.735081:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948919.735087:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948919.735092:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948919.735096:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948919.735100:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948919.735105:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948919.735111:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000553902b5. 00000100:00000200:0.0:1784948919.735118:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289661696, offset 224 00000400:00000200:0.0:1784948919.735127:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948919.735141:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948919.735152:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3406:3406:233:4294967295] 192.168.202.137@tcp LPNI seq info [2193:2193:8:4294967295] 00000400:00000200:0.0:1784948919.735172:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948919.735180:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948919.735188:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002aeee8e9. 00000800:00000200:0.0:1784948919.735197:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948919.735209:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948919.735215:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002aeee8e9 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948919.735239:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948919.735246:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948919.735250:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948919.735253:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948919.735257:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948919.735263:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000299dc118 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:580/0 lens 224/224 e 0 to 0 dl 1784948930 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948919.735279:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000299dc118 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289661696:12345-192.168.202.137@tcp:400: Request processed in 616us (1233us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948919.735293:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 710 00000100:00000040:0.0:1784948919.735298:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948919.735303:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948919.735306:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948919.735313:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bcdae09a. 00000020:00000010:0.0:1784948919.735320:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948919.735324:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000043c3e1d8. 00000020:00000040:0.0:1784948919.735330:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948919.735336:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948919.735489:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948919.735496:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002aeee8e9. 00000400:00000200:2.0:1784948919.735502:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948919.735510:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948919.735514:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948919.735516:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553902b5 00000400:00000010:2.0:1784948919.735523:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000553902b5. 00000100:00000001:2.0:1784948919.735528:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784948919.735530:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948919.735533:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948919.735535:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948919.735542:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948919.735551:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948919.735565:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438100 00000400:00000200:0.0:1784948919.735573:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8801 [1] + 224 00000800:00000001:0.0:1784948919.735579:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948919.735588:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948919.735590:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948919.735594:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948919.735599:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948919.735603:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948919.735613:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948919.735657:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948919.735665:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948919.735668:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948919.735753:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948919.735759:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948919.735766:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948919.735769:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948919.735772:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000de368614. 00000400:00000200:3.0:1784948919.735780:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:3.0:1784948919.735782:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000100:00000001:3.0:1784948919.735786:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948919.735793:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948919.735805:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948919.735813:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948919.735815:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948919.735818:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948919.735821:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948919.735823:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948919.735825:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948919.735828:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948919.735835:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=86 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948919.735839:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=86 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948919.735842:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948919.735844:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948919.735847:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948919.735851:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948919.735858:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.735862:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948919.735864:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948919.735865:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948919.735867:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948919.735870:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948919.735877:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948919.735880:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948919.735882:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948919.735885:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948919.735893:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008d623923 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289661696:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948919.735898:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948919.735900:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948919.735902:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008d623923 x1871651289661696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948919.735935:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948919.735937:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948919.735940:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 02000000:00000001:3.0:1784948919.735945:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948919.735947:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948919.735948:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948919.735951:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948919.735954:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000cee99234. 02000000:00000001:3.0:1784948919.735957:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948919.735959:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948919.735960:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008d623923. 00000100:00000001:3.0:1784948919.735964:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948919.735966:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948919.735967:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948919.735972:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948919.735973:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948919.735975:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948919.735978:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948919.776014:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305859115, at 4305859116 00000100:00000010:0.0:1784948919.776032:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000006204bf6f. 00000100:00000040:1.0:1784948919.776047:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305859115, at 4305859116 00000100:00000040:2.0:1784948919.776050:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305859115, at 4305859116 00000100:00000040:3.0:1784948919.776050:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305859115, at 4305859116 00000100:00000010:1.0:1784948919.776069:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00000100:00000010:3.0:1784948919.776074:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000100:00000010:2.0:1784948919.776082:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000029366a5. 00010000:00000001:2.0:1784948920.564023:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948920.564035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948920.564045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948920.564049:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948920.564054:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948920.564059:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948920.564065:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948920.564068:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948920.564071:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948920.564075:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948920.564213:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948920.564223:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948920.564228:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948920.756012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948920.756027:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948920.756029:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948920.756053:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948920.756058:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948920.756062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948920.756066:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948921.588097:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948921.588110:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948921.588120:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948921.588121:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948921.588221:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948921.588252:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948921.588260:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948921.780080:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948921.780104:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948921.780108:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948921.780115:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948921.780119:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948921.780123:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948921.780126:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784948921.972139:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:0.0:1784948921.972149:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005a1c05a3. 00000400:00000200:0.0:1784948921.972163:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948921.972172:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784948921.972183:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11195 health :900 00000400:00000200:0.0:1784948921.972190:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948921.972196:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:0.0:1784948921.972202:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:3.0:1784948921.972296:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948921.972319:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:3.0:1784948921.972334:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948921.972338:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948921.972343:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000010:3.0:1784948921.972348:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000400:00000200:3.0:1784948921.972358:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948921.972367:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:3.0:1784948921.972373:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000041892d66. 00000400:00000200:3.0:1784948921.972381:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:3.0:1784948921.972385:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:3.0:1784948921.972389:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00010000:00000001:2.0:1784948922.612073:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948922.612087:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948922.612099:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948922.612101:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948922.612259:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948922.612270:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948922.612278:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948922.804047:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948922.804067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948922.804070:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948922.804076:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948922.804080:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948922.804085:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948922.804088:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948922.996068:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948922.996080:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000048fd39a3. 00000400:00000010:1.0:1784948922.996095:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dbce179e. 00000400:00000010:1.0:1784948922.996102:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000010:1.0:1784948922.996111:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e360eb6a. 00000400:00000200:1.0:1784948922.996117:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948922.996134:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948922.996142:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948922.996146:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3407:3407:234:4294967295] 192.168.202.137@tcp LPNI seq info [2194:2194:8:4294967295] 00000400:00000200:1.0:1784948922.996155:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948922.996164:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948922.996171:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948922.996176:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000800:00000200:1.0:1784948922.996184:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948922.996195:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948922.996201:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000031deb17e type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948922.996228:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11197 health :900 00000800:00000200:2.0:1784948922.996473:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948922.996489:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:2.0:1784948922.996508:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948922.996525:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784948922.996539:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784948922.996545:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948922.996547:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784948922.996570:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948922.996577:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948922.996587:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948922.996626:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784948922.996655:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784948922.996670:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784948922.996687:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948922.996696:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3408:3408:234:4294967295] 192.168.202.137@tcp LPNI seq info [2195:2195:8:4294967295] 00000400:00000200:0.0:1784948922.996714:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784948922.996723:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948922.996732:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008746b1e6. 00000800:00000200:0.0:1784948922.996742:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948922.996752:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948922.996758:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008746b1e6 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784948922.996933:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948922.997068:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948922.997073:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784948922.997079:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008746b1e6. 00000400:00000200:0.0:1784948922.997093:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948922.997098:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948922.997108:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784948922.997111:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948922.997112:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948922.997128:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948922.997146:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948922.997172:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8815 00000800:00000001:2.0:1784948922.997183:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948922.997199:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948922.997203:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948922.997208:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948922.997218:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:2.0:1784948922.997223:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000400:00000200:2.0:1784948922.997228:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948922.997232:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000048fd39a3. 00000400:00000010:2.0:1784948922.997238:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dbce179e. 00000800:00000001:2.0:1784948922.997253:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948922.997257:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000100:1.0:1784948923.316257:0:344013:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.7:988 00000400:00000100:2.0:1784948923.316269:0:344015:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.11:988 00000400:00000100:3.0:1784948923.316288:0:344012:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.15:988 00000400:00000100:0.0:1784948923.316336:0:343911:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.26:988 00000400:00000100:1.0:1784948923.316482:0:344013:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.7@tcp at host 111.222.173.7:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784948923.316484:0:344015:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.11@tcp at host 111.222.173.11:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784948923.316484:0:344012:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.15@tcp at host 111.222.173.15:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:0.0:1784948923.316485:0:343911:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.26@tcp at host 111.222.173.26:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948923.316573:0:344012:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.15@tcp: down 00000400:00000200:0.0:1784948923.316576:0:343911:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.26@tcp: down 00000400:00000200:1.0:1784948923.316576:0:344013:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.7@tcp: down 00000400:00000200:2.0:1784948923.316576:0:344015:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.11@tcp: down 00000400:00000200:1.0:1784948923.316640:0:344013:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948923.316652:0:344015:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1784948923.316667:0:344015:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000800:00000200:0.0:1784948923.316670:0:343911:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.173.26@tcp down 00000400:00000200:3.0:1784948923.316674:0:344012:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:2.0:1784948923.316680:0:344015:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000820f02b3. 00000400:00000200:2.0:1784948923.316695:0:344015:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:3.0:1784948923.316701:0:344012:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000400:00000200:2.0:1784948923.316712:0:344015:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000800:00000010:3.0:1784948923.316714:0:344012:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000042e55fb6. 00000400:00000200:2.0:1784948923.316729:0:344015:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948923.316737:0:344012:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000100:0.0:1784948923.316739:0:344008:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.3:988 00000400:00000200:2.0:1784948923.316744:0:344015:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:2.0:1784948923.316769:0:344015:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:3.0:1784948923.316775:0:344012:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000100:0.0:1784948923.316801:0:344008:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.3@tcp at host 111.222.173.3:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784948923.316820:0:344012:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948923.316821:0:344008:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.3@tcp: down 00000400:00000200:0.0:1784948923.316839:0:344008:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:2.0:1784948923.316849:0:343956:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.1:988 00000400:00000200:3.0:1784948923.316856:0:344012:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000100:1.0:1784948923.316862:0:344010:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.13:988 00000800:00000100:0.0:1784948923.316869:0:344008:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000400:00000200:3.0:1784948923.316882:0:344012:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000800:00000010:0.0:1784948923.316884:0:344008:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002a753d9d. 00000400:00000200:0.0:1784948923.316983:0:344008:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000100:2.0:1784948923.316985:0:343956:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.1@tcp at host 111.222.173.1:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784948923.317014:0:344008:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948923.317022:0:343956:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.1@tcp: down 00000400:00000200:2.0:1784948923.317035:0:343956:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:1.0:1784948923.317037:0:344010:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.13@tcp at host 111.222.173.13:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784948923.317044:0:344008:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000100:2.0:1784948923.317055:0:343956:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000400:00000200:0.0:1784948923.317067:0:344008:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:1.0:1784948923.317072:0:344010:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.13@tcp: down 00000800:00000010:2.0:1784948923.317072:0:343956:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000036e798e7. 00000400:00000200:0.0:1784948923.317084:0:344008:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:2.0:1784948923.317091:0:343956:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948923.317099:0:343956:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948923.317104:0:344010:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784948923.317115:0:344010:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000800:00000010:1.0:1784948923.317121:0:344010:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a12a46b5. 00000400:00000200:2.0:1784948923.317124:0:343956:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948923.317137:0:343956:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:1.0:1784948923.317139:0:344010:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000100:3.0:1784948923.317167:0:344014:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.5:988 00000400:00000200:2.0:1784948923.317178:0:343956:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:1.0:1784948923.317185:0:344010:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000100:0.0:1784948923.317186:0:344011:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.9:988 00000400:00000200:1.0:1784948923.317215:0:344010:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948923.317231:0:344010:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:1.0:1784948923.317254:0:344010:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000100:3.0:1784948923.317275:0:344014:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.5@tcp at host 111.222.173.5:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948923.317277:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948923.317301:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000200:3.0:1784948923.317303:0:344014:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.5@tcp: down 00000400:00000100:0.0:1784948923.317315:0:344011:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.9@tcp at host 111.222.173.9:988 took too long: that node may be hung or experiencing high load. 00000400:00000001:2.0:1784948923.317316:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:3.0:1784948923.317316:0:344014:0:(router.c:1888:lnet_notify()) Old news 00000400:00000001:2.0:1784948923.317325:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948923.317332:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000800:00000100:3.0:1784948923.317344:0:344014:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.5@tcp 00000400:00000010:2.0:1784948923.317345:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000400:00000200:0.0:1784948923.317351:0:344011:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.9@tcp: down 00000400:00000200:2.0:1784948923.317362:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000010:3.0:1784948923.317366:0:344014:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000057c7ab4. 00000400:00000200:0.0:1784948923.317371:0:344011:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784948923.317378:0:344014:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948923.317388:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000800:00000100:0.0:1784948923.317396:0:344011:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.9@tcp 00000400:00000010:2.0:1784948923.317404:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001c9428b0. 00000400:00000200:3.0:1784948923.317405:0:344014:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: NETWORK_TIMEOUT 00000800:00000010:0.0:1784948923.317413:0:344011:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000635133da. 00000400:00000200:2.0:1784948923.317423:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:3.0:1784948923.317427:0:344014:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948923.317429:0:344011:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948923.317432:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:3.0:1784948923.317450:0:344014:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:2.0:1784948923.317461:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000400:00000200:0.0:1784948923.317467:0:344011:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948923.317478:0:344014:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:2.0:1784948923.317494:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000200:0.0:1784948923.317505:0:344011:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000001:2.0:1784948923.317506:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948923.317518:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948923.317528:0:344011:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:2.0:1784948923.317528:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:2.0:1784948923.317544:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000400:00000200:0.0:1784948923.317553:0:344011:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:2.0:1784948923.317554:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948923.317570:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:2.0:1784948923.317573:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001df3b40e. 00000400:00000200:2.0:1784948923.317579:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:2.0:1784948923.317607:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:2.0:1784948923.317616:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000400:00000200:2.0:1784948923.317626:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:2.0:1784948923.317630:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948923.317633:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948923.317635:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000400:00000010:2.0:1784948923.317637:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000400:00000200:2.0:1784948923.317641:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948923.317643:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:2.0:1784948923.317646:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000bbdbdf4. 00000400:00000200:2.0:1784948923.317650:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:2.0:1784948923.317652:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:2.0:1784948923.317655:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:2.0:1784948923.317659:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:2.0:1784948923.317662:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948923.317663:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948923.317665:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784948923.317667:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000400:00000200:2.0:1784948923.317669:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948923.317671:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:2.0:1784948923.317673:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000467d4d37. 00000400:00000200:2.0:1784948923.317676:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:2.0:1784948923.317679:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:2.0:1784948923.317681:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000400:00000200:2.0:1784948923.317685:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:2.0:1784948923.317687:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948923.317688:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948923.317690:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:2.0:1784948923.317692:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:2.0:1784948923.317694:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948923.317696:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:2.0:1784948923.317699:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000015713dee. 00000400:00000200:2.0:1784948923.317704:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:2.0:1784948923.317707:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:2.0:1784948923.317709:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000400:00000200:2.0:1784948923.317713:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 00000400:00000001:2.0:1784948923.317715:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948923.317716:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948923.317718:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000010:2.0:1784948923.317721:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000400:00000200:2.0:1784948923.317723:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948923.317726:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:2.0:1784948923.317729:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000244a9cae. 00000400:00000200:2.0:1784948923.317734:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-110 00000400:00000200:2.0:1784948923.317736:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -110 00000400:00000200:2.0:1784948923.317739:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:2.0:1784948923.317742:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000001:2.0:1784948923.317745:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948923.317746:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948923.317748:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:2.0:1784948923.317749:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000400:00000200:2.0:1784948923.317751:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948923.317754:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:2.0:1784948923.317756:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000075318cf7. 00000400:00000200:2.0:1784948923.317760:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:2.0:1784948923.317762:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000400:00000200:2.0:1784948923.317765:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00010000:00000001:2.0:1784948923.636102:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948923.636117:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948923.636128:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948923.636130:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948923.636218:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948923.636227:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948923.636232:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948923.828068:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948923.828090:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948923.828094:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948923.828102:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948923.828107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948923.828113:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948923.828117:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948924.660048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948924.660061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948924.660070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948924.660072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948924.660217:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948924.660226:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948924.660231:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948924.852020:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948924.852037:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784948924.852055:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948899/real 1784948899] req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948924.852087:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:2.0:1784948924.852088:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948924.852093:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784948924.852097:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000010:2.0:1784948924.852100:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000047084e78. 00000400:00000200:2.0:1784948924.852110:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000100:00000010:3.0:1784948924.852111:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b16a92e. 00000400:00000010:2.0:1784948924.852116:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000020:00000040:3.0:1784948924.852119:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948924.852124:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784948924.852128:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1784948924.852129:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948924.852134:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1784948924.852141:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948924.852144:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.852150:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948924.852153:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.852156:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948924.852158:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784948924.852159:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00100000:2.0:1784948924.852161:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.852171:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948924.852172:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948924.852180:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948924.852181:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948924.852185:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.852191:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.852196:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948924.852199:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948924.852200:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948924.852204:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948924.852208:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948924 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948924.852214:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b16a92e] to pc [ptlrpcd_00_03+3] req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.852235:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784948924.852243:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.852248:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948924.852251:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.852254:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784948924.852256:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784948924.852259:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1784948924.852278:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948924.852281:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948924.852285:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784948924.852289:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1784948924.852290:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00080000:3.0:1784948924.852293:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784948924.852295:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1784948924.852301:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:2.0:1784948924.852305:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948924 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1784948924.852309:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11171 00000100:00100000:2.0:1784948924.852321:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289660544:192.168.202.137@tcp:8: 00000400:00000200:3.0:1784948924.852326:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:3.0:1784948924.852331:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11170 00000400:00000200:3.0:1784948924.852347:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000400:00000200:3.0:1784948924.852350:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000040:3.0:1784948924.852353:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948924.852358:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948924.852361:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948924.852363:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000001:3.0:1784948924.852367:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948924.852371:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:3.0:1784948924.852375:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948924.852379:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:3.0:1784948924.852384:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:3.0:1784948924.852388:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:3.0:1784948924.852392:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948924.852395:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948924.852396:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:3.0:1784948924.852398:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:3.0:1784948924.852401:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:1.0:1784948924.852403:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784948924.852409:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000001:0.0:1784948924.852410:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948924.852411:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1784948924.852420:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:1.0:1784948924.852421:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948924.852424:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948924.852430:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948924.852431:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948924.852435:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948924.852436:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948924.852437:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1784948924.852443:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.852443:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948924.852448:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784948924.852449:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784948924.852452:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ddd830de x1871651289660544/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948924 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948924.852455:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1784948924.852456:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948924.852464:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1784948924.852467:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.852468:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784948924.852470:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948924.852474:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948924.852481:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d02c7fef. 00000100:00000010:3.0:1784948924.852481:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000058a1380c. 00000100:00000001:1.0:1784948924.852484:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1784948924.852489:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=4 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:1.0:1784948924.852491:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948924.852497:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1784948924.852500:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1784948924.852502:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 02000000:00000001:1.0:1784948924.852502:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1784948924.852503:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948924.852514:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784948924.852516:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948924.852517:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1784948924.852521:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784948924.852525:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1784948924.852527:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 02000000:00000001:1.0:1784948924.852529:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948924.852530:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:2.0:1784948924.852531:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000009265644e. 02000000:00000001:3.0:1784948924.852539:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948924.852547:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289661824:0@lo:400: 02000000:00000001:2.0:1784948924.852549:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:3.0:1784948924.852551:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005b888794. 00000100:00000001:3.0:1784948924.852564:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948924.852565:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948924.852566:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000010:0.0:1784948924.852573:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000043c3e1d8. 00000100:00000010:2.0:1784948924.852573:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ddd830de. 00000100:00100000:3.0:1784948924.852574:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1784948924.852593:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948924.852598:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.852599:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948924.852608:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948924.852632:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948924.852633:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948924.852639:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000100:00000001:3.0:1784948924.852640:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:1.0:1784948924.852641:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000035b54a67. 00000100:00000001:2.0:1784948924.852642:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948924.852646:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948924.852657:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.852657:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948924.852662:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000058a1380c] to pc [ptlrpcd_rcv+-1] req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948924.852665:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003ad675ec. 00000100:00000001:2.0:1784948924.852667:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784948924.852670:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000789ee428. 00000100:00000001:2.0:1784948924.852676:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948924.852688:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000400:00000010:1.0:1784948924.852700:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000001:3.0:1784948924.852701:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948924.852707:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948924.852710:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784948924.852711:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289661824, portal 25 00000400:00000200:0.0:1784948924.852713:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:2.0:1784948924.852718:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948924.852726:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948924.852726:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948924.852727:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948924.852733:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482521600 : -115544227030016 : ffff96e9c3645c00) 00000400:00000200:0.0:1784948924.852734:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1784948924.852743:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948924.852751:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:1.0:1784948924.852756:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784948924.852765:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003492edb3. 00000400:00000200:0.0:1784948924.852769:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1784948924.852781:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:0.0:1784948924.852789:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:3.0:1784948924.852794:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784948924.852803:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784948924.852812:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.852817:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784948924.852818:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3409:3409:241:4294967295] 1.2.3.4@tcp LPNI seq info [101:101:8:4294967295] 02000000:00000010:3.0:1784948924.852821:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:2.0:1784948924.852825:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948924.852840:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948924.852842:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948924.852843:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948924.852843:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948924.852862:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948924.852864:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:3.0:1784948924.852866:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1784948924.852871:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000058a1380c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289661952:1.2.3.4@tcp:8: 00000400:00000010:1.0:1784948924.852872:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000400:00000200:0.0:1784948924.852878:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000200:1.0:1784948924.852888:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289661824, offset 0 00000100:00000001:2.0:1784948924.852892:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784948924.852898:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.852914:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948924.852922:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:0.0:1784948924.852937:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:2.0:1784948924.852956:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948924.852960:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948924.852960:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948924.852961:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948924.852965:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.852968:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948924.852970:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948924.852973:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003492edb3] to pc [ptlrpcd_00_00+0] req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.852976:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.852980:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948924.852984:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948924.852986:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:0.0:1784948924.852988:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008746b1e6. 00000100:00000001:2.0:1784948924.852992:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948924.853027:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:1.0:1784948924.853028:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438180 00000100:00000001:3.0:1784948924.853029:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853040:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.853047:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948924.853048:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11201) 00000100:00000001:2.0:1784948924.853052:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1784948924.853053:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7e0d [2] + 0 00000800:00000200:0.1:1784948924.853065:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:2.0:1784948924.853067:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948924.853068:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11201) 00000400:00000010:1.0:1784948924.853072:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000da8fa306. 00000400:00000200:1.0:1784948924.853088:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000004ba703f8 00000400:00000200:0.0:1784948924.853096:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:2.0:1784948924.853097:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948924.853098:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948924.853105:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948924.853107:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948924.853113:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948924.853116:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784948924.853118:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 00000400:00000200:1.0:1784948924.853124:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:2.0:1784948924.853127:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948924.853138:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948924.853149:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289662080:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784948924.853150:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ba703f8 00000400:00000010:1.0:1784948924.853155:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000004ba703f8. 00000100:00000001:2.0:1784948924.853156:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948924.853162:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948924.853164:0:337121:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:2.0:1784948924.853164:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948924.853166:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000200:1.0:1784948924.853168:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000010:2.0:1784948924.853172:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 00000100:00000040:1.0:1784948924.853175:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000022e78e62 x1871651289661824 msgsize 224 02000000:00000001:2.0:1784948924.853181:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948924.853187:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000010:2.0:1784948924.853190:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005cc7db3b. 00000100:00000040:1.0:1784948924.853191:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000010:2.0:1784948924.853200:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000100:00000200:2.0:1784948924.853204:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289662080, portal 10 00000100:00000001:2.0:1784948924.853206:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948924.853208:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554958848 : -115544154592768 : ffff96e9c7b5aa00) 00000100:00000001:1.0:1784948924.853210:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:2.0:1784948924.853216:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948924.853221:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000100:00000001:2.0:1784948924.853223:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784948924.853225:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000100:00000040:2.0:1784948924.853228:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948924.853240:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000100:00000001:1.0:1784948924.853242:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948924.853245:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289662080, offset 0 00000100:00000200:1.0:1784948924.853250:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948924.853257:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948924.853268:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948924.853274:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:1.0:1784948924.853278:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948924.853283:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784948924.853289:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948924.853290:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948924.853295:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784948924.853300:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1784948924.853301:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289661824 00000400:00000200:2.0:1784948924.853310:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784948924.853311:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948924.853325:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784948924.853326:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784948924.853337:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948924.853347:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3410:3410:240:4294967295] 192.168.202.137@tcp LPNI seq info [2196:2196:8:4294967295] 02000000:00000001:0.0:1784948924.853348:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948924.853364:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289661824 00000020:00000001:0.0:1784948924.853373:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948924.853381:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:2.0:1784948924.853382:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:0.0:1784948924.853391:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948924.853397:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948924.853398:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948924.853400:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784948924.853417:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948924.853420:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305864235 00000800:00000010:2.0:1784948924.853420:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b9b32818. 00000020:00000001:0.0:1784948924.853430:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784948924.853435:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:0.0:1784948924.853440:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:2.0:1784948924.853440:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948924.853447:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:0.0:1784948924.853454:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:1.0:1784948924.853458:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305864235 00000100:00000001:0.0:1784948924.853462:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1784948924.853463:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000010:1.0:1784948924.853469:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000010:0.0:1784948924.853479:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000800:00000200:2.0:1784948924.853480:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b9b32818 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948924.853481:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948924.853495:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305864235 00000100:00000010:1.0:1784948924.853502:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000020:00000010:0.0:1784948924.853504:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001879354a. 00000100:00000001:1.0:1784948924.853508:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948924.853508:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948924.853512:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bcdae09a. 00000100:00000001:1.0:1784948924.853514:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305864235 00000100:00000040:2.0:1784948924.853516:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2098 usecs (3 jiffies) 00000100:00000001:2.0:1784948924.853526:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305864196 00000100:00000001:2.0:1784948924.853530:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305864235 00000100:00000010:1.0:1784948924.853533:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000020:00000010:0.0:1784948924.853535:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784948924.853535:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305864196 00000100:00000001:1.0:1784948924.853549:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948924.853551:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305864235 00000100:00000001:2.0:1784948924.853558:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305864196 00000100:00000040:0.0:1784948924.853563:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948924.853563:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305864235 00000100:00000001:1.0:1784948924.853564:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948924.853574:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948924.853587:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853588:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305864196 00000100:00000001:0.0:1784948924.853598:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948924.853600:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853600:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305864235 00000100:00000001:0.0:1784948924.853602:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853620:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948924.853622:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853625:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948924.853629:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853631:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853648:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.853655:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948924.853658:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948924.853660:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.853661:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948924.853662:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.853665:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948924.853666:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948924.853685:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948924.853688:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948924.853700:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 735 00000100:00000040:0.0:1784948924.853704:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948924.853708:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742036560 : -115538967515056 : ffff96eafce23050) 00000100:00000040:0.0:1784948924.853717:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000022e78e62 x1871651289661824/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:585/0 lens 224/0 e 0 to 0 dl 1784948935 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948924.853729:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948924.853734:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784948924.853736:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948924.853739:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000800:00000200:3.0:1784948924.853743:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00100000:0.0:1784948924.853755:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000022e78e62 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289661824:12345-0@lo:400: 00000800:00000010:3.0:1784948924.853756:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b9b32818. 00000800:00000001:2.0:1784948924.853759:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948924.853769:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1784948924.853770:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289661824 00000800:00000001:2.0:1784948924.853780:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948924.853785:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:2.0:1784948924.853786:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948924.853793:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784948924.853795:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000001:2.0:1784948924.853798:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948924.853800:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948924.853800:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000200:2.0:1784948924.853817:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:3.0:1784948924.853817:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000020:00000001:0.0:1784948924.853819:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948924.853832:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784948924.853835:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784948924.853840:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:2.0:1784948924.853845:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438280 00000100:00000200:3.0:1784948924.853849:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948924.853850:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948924.853863:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 160304 00000020:00000001:0.0:1784948924.853865:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948924.853874:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784948924.853882:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784948924.853886:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948924.853888:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784948924.853894:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:3.0:1784948924.853894:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948924.853902:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948924.853936:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948924.853938:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784948924.853949:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948924.853967:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:2.0:1784948924.853968:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:0.0:1784948924.853976:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948924.853979:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948924.853983:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000010:0.0:1784948924.853984:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000867cea4b. 02000000:00000001:0.0:1784948924.853991:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948924.853992:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000078cad088. 00000100:00000001:0.0:1784948924.853999:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948924.854002:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000078cad088 x1871651289662080 msgsize 224 00000020:00000001:0.0:1784948924.854012:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948924.854027:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289661824 00000100:00100000:2.0:1784948924.854027:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:0.0:1784948924.854043:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948924.854053:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000022e78e62 x1871651289661824/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:585/0 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.854053:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:2.0:1784948924.854062:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948924.854072:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948924.854074:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948924.854082:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948924.854087:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=96 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948924.854090:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948924.854097:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289662080 00000100:00000001:0.0:1784948924.854098:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784948924.854100:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:0.0:1784948924.854105:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948924.854107:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948924.854117:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948924.854118:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948924.854132:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948924.854133:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948924.854136:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948924.854140:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289662080 00000100:00000001:0.0:1784948924.854143:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1784948924.854146:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:0.0:1784948924.854153:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000040:1.0:1784948924.854153:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948924.854167:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784948924.854169:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000020:00000040:1.0:1784948924.854173:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:0.0:1784948924.854178:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289661824, offset 224 00000020:00000001:1.0:1784948924.854179:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948924.854196:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948924.854203:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784948924.854204:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:1.0:1784948924.854214:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784948924.854221:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948924.854233:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000400:00000200:0.0:1784948924.854250:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:1.0:1784948924.854252:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000043d62367. 00000020:00000010:1.0:1784948924.854263:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008329b0c3. 00000400:00000200:0.0:1784948924.854269:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438180 00000020:00000010:1.0:1784948924.854284:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000400:00000200:0.0:1784948924.854294:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x881d [1] + 224 00000100:00000040:1.0:1784948924.854302:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948924.854306:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:0.0:1784948924.854308:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948924.854309:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948924.854315:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948924.854321:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948924.854335:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948924.854336:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784948924.854343:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948924.854346:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948924.854347:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948924.854367:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948924.854368:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948924.854392:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948924.854396:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948924.854403:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 711 00000100:00000040:1.0:1784948924.854409:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948924.854410:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948924.854419:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470018816 : -115544239532800 : ffff96e9c2a59500) 00000400:00000200:0.0:1784948924.854429:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000100:00000040:1.0:1784948924.854432:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000078cad088 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:585/0 lens 224/0 e 0 to 0 dl 1784948935 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948924.854435:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784948924.854449:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948924.854450:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948924.854453:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000010:1.0:1784948924.854456:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:0.0:1784948924.854463:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948924.854464:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948924.854479:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00000040:0.0:1784948924.854483:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784948924.854487:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784948924.854490:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00100000:1.0:1784948924.854491:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000078cad088 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289662080:12345-192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948924.854495:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784948924.854496:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948924.854499:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289662080 00000400:00000001:2.0:1784948924.854507:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1784948924.854509:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:1.0:1784948924.854509:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000001:2.0:1784948924.854513:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784948924.854517:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:2.0:1784948924.854518:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000789ee428. 00000100:00000040:0.0:1784948924.854523:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000022e78e62 x1871651289661824/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:585/0 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948924.854524:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948924.854529:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000020:00000001:1.0:1784948924.854537:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:2.0:1784948924.854538:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000020:00000001:1.0:1784948924.854543:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784948924.854555:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1784948924.854557:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948924.854565:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948924.854570:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948924.854596:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00100000:0.0:1784948924.854601:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000022e78e62 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289661824:12345-0@lo:400: Request processed in 839us (1393us total) trans 0 rc 0/0 00000020:00000001:1.0:1784948924.854603:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948924.854611:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948924.854624:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948924.854633:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948924.854637:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948924.854645:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fd4da236. 00000100:00100000:0.0:1784948924.854646:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 735 00000100:00000200:2.0:1784948924.854656:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948924.854657:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948924.854659:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948924.854665:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948924.854679:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:1.0:1784948924.854679:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.854681:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948924.854696:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784948924.854697:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1784948924.854698:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289662080 00010000:00000001:1.0:1784948924.854707:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784948924.854708:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:0.0:1784948924.854709:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bcdae09a. 02000000:00000001:2.0:1784948924.854716:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00010000:00000200:1.0:1784948924.854719:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000078cad088 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:585/0 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.854721:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:0.0:1784948924.854722:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784948924.854727:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948924.854737:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001879354a. 02000000:00000001:2.0:1784948924.854737:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948924.854745:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948924.854757:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948924.854760:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=96 v=5 (5 5 5 5) 00000020:00000040:0.0:1784948924.854761:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:1.0:1784948924.854766:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=91 v=5 (1 1 1 1) 02000000:00000001:0.0:1784948924.854781:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:2.0:1784948924.854781:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=96 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948924.854783:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:0.0:1784948924.854797:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000867cea4b. 00000100:00000001:2.0:1784948924.854797:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000040:1.0:1784948924.854800:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948924.854808:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948924.854811:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784948924.854812:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00010000:00000001:2.0:1784948924.854814:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784948924.854821:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948924.854822:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00010000:00000001:2.0:1784948924.854822:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948924.854830:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948924.854840:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948924.854840:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948924.854841:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:1.0:1784948924.854855:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948924.854859:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000094ceb1ba. 00000100:00000040:2.0:1784948924.854860:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948924.854861:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000200:1.0:1784948924.854880:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289662080, offset 224 00000100:00000001:2.0:1784948924.854880:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948924.854882:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000042f7d5bc. 00000100:00000001:2.0:1784948924.854887:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948924.854900:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948924.854903:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784948924.854943:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948924.854951:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948924.854963:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3411:3411:240:4294967295] 192.168.202.137@tcp LPNI seq info [2197:2197:8:4294967295] 00000100:00100000:2.0:1784948924.854964:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289661824:0@lo:400: 00000100:00000001:2.0:1784948924.854977:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.854983:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948924.854985:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:2.0:1784948924.854989:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b16a92e x1871651289661824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784948924.854999:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948924.855006:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000010:1.0:1784948924.855007:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 02000000:00000001:2.0:1784948924.855009:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948924.855013:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000035b54a67. 00000800:00000200:1.0:1784948924.855018:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:2.0:1784948924.855023:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000200:1.0:1784948924.855024:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000020:00000001:2.0:1784948924.855028:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000800:00000200:1.0:1784948924.855029:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000020:00000040:2.0:1784948924.855031:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784948924.855036:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948924.855038:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 02000000:00000001:2.0:1784948924.855042:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948924.855043:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948924.855044:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b16a92e. 00000100:00000001:1.0:1784948924.855045:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.855048:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:1.0:1784948924.855051:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948924.855052:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948924.855057:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:2.0:1784948924.855057:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948924.855062:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.855065:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1784948924.855066:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784948924.855071:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948924.855075:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000078cad088 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:585/0 lens 224/224 e 0 to 0 dl 1784948935 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.855077:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.855081:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948924.855092:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000078cad088 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289662080:12345-192.168.202.137@tcp:400: Request processed in 639us (1088us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948924.855100:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 711 00000100:00000040:1.0:1784948924.855104:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948924.855107:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948924.855109:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948924.855113:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008329b0c3. 00000020:00000010:1.0:1784948924.855118:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948924.855121:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000043d62367. 00000020:00000040:1.0:1784948924.855125:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948924.855127:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948924.855164:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948924.855168:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000400:00000200:3.0:1784948924.855174:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948924.855181:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948924.855187:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000400:00000010:3.0:1784948924.855189:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000100:00000001:3.0:1784948924.855192:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948924.855194:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948924.855197:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948924.855217:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948924.855221:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948924.855225:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948924.855232:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948924.855243:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438280 00000400:00000200:0.0:1784948924.855250:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8825 [1] + 224 00000800:00000001:0.0:1784948924.855256:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948924.855266:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948924.855269:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948924.855273:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948924.855278:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948924.855283:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948924.855294:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948924.855315:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948924.855322:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948924.855324:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.855348:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.855362:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948924.855367:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948924.855369:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948924.855371:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005cc7db3b. 00000400:00000200:2.0:1784948924.855376:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:2.0:1784948924.855378:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000100:00000001:2.0:1784948924.855380:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948924.855386:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948924.855394:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.855400:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948924.855401:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.855402:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948924.855404:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948924.855405:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948924.855407:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948924.855409:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948924.855413:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=91 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948924.855416:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=91 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948924.855418:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948924.855419:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948924.855421:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948924.855423:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948924.855429:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.855431:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948924.855433:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948924.855434:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948924.855435:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948924.855437:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.855443:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948924.855445:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948924.855446:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948924.855448:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948924.855453:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289662080:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948924.855456:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.855457:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948924.855458:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003492edb3 x1871651289662080/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948940 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948924.855463:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948924.855464:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948924.855465:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 02000000:00000001:2.0:1784948924.855467:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948924.855469:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948924.855470:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948924.855472:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948924.855473:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:2.0:1784948924.855475:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948924.855476:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948924.855478:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003492edb3. 00000100:00000001:2.0:1784948924.855480:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948924.855481:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.855482:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948924.855485:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948924.855486:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948924.855487:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948924.855489:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948924.896005:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305864235, at 4305864236 00000100:00000040:0.0:1784948924.896023:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305864235, at 4305864236 00000100:00000010:3.0:1784948924.896024:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:1.0:1784948924.896025:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305864235, at 4305864236 00000100:00000040:2.0:1784948924.896026:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305864235, at 4305864236 00000100:00000010:1.0:1784948924.896049:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:0.0:1784948924.896058:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:2.0:1784948924.896062:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000400:00000200:1.0:1784948925.044095:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948925.044112:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a2236612. 00000400:00000010:1.0:1784948925.044128:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000043d62367. 00000400:00000010:1.0:1784948925.044137:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000400:00000010:1.0:1784948925.044150:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008e22c077. 00000400:00000200:1.0:1784948925.044157:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948925.044184:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948925.044193:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948925.044199:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3412:3412:240:4294967295] 192.168.202.137@tcp LPNI seq info [2198:2198:8:4294967295] 00000400:00000200:1.0:1784948925.044225:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948925.044236:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948925.044245:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948925.044252:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000800:00000200:1.0:1784948925.044264:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948925.044299:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948925.044305:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948925.044331:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11199 health :700 00000800:00000200:2.0:1784948925.044636:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948925.044644:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000400:00000200:2.0:1784948925.044658:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948925.044667:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784948925.044675:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:3.0:1784948925.044676:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948925.044680:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948925.044721:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948925.044730:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948925.044739:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948925.044753:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948925.044774:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784948925.044789:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784948925.044806:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948925.044814:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3413:3413:240:4294967295] 192.168.202.137@tcp LPNI seq info [2199:2199:8:4294967295] 00000400:00000200:3.0:1784948925.044828:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784948925.044836:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948925.044843:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784948925.044851:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948925.044859:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948925.044864:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948925.044997:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784948925.045078:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948925.045086:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:3.0:1784948925.045094:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948925.045108:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948925.045126:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948925.045126:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948925.045144:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948925.045151:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948925.045160:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948925.045172:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948925.045187:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8839 00000800:00000001:2.0:1784948925.045193:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948925.045208:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948925.045211:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948925.045215:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948925.045221:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:2.0:1784948925.045225:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000400:00000200:2.0:1784948925.045229:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948925.045233:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a2236612. 00000400:00000010:2.0:1784948925.045237:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000043d62367. 00000800:00000001:2.0:1784948925.045244:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948925.045247:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948925.684031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948925.684043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948925.684051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948925.684054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948925.684145:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948925.684153:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948925.684159:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948925.876087:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948925.876102:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1784948925.876123:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948924/real 0] req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948925 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948925.876147:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.876152:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948925.876155:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948925.876162:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948925 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948925.876176:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.876191:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948925.876194:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948925.876199:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948925.876205:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948925.876209:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948925.876216:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948925 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948925.876239:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1784948925.876245:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.876248:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948925.876251:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.876256:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1784948925.876260:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1784948925.876267:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1784948925.876309:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:3.0:1784948925.876316:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1784948925.876325:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948925 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948925.876339:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000058a1380c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289661952:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784948925.876348:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.876353:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948925.876359:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000058a1380c x1871651289661952/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948925 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948925.876371:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1784948925.876387:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948925.876392:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:3.0:1784948925.876396:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:0.0:1784948925.876403:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:3.0:1784948925.876404:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000005b888794. 00000100:00000001:0.0:1784948925.876414:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:3.0:1784948925.876418:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000010:0.0:1784948925.876424:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 02000000:00000001:3.0:1784948925.876424:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948925.876430:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000058a1380c. 00000020:00000040:0.0:1784948925.876433:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948925.876442:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948925.876443:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1784948925.876447:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948925.876449:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.876456:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948925.876457:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948925.876462:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948925.876467:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948925.876469:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002aeee8e9. 00000100:00000001:3.0:1784948925.876472:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948925.876478:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.876481:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948925.876492:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948925.876505:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948925.876508:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948925.876515:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59e6b47] to pc [ptlrpcd_00_01+1] req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948925.876563:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948925.876584:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948925.876590:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784948925.876596:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784948925.876600:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948925.876607:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1784948925.876610:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784948925.876615:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11171 00000100:00100000:2.0:1784948925.876618:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:3.0:1784948925.876620:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948925.876624:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.876639:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.876644:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1784948925.876646:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784948925.876648:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948925.876657:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784948925.876658:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11196 00000100:00000001:2.0:1784948925.876660:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948925.876666:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784948925.876670:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000040:2.0:1784948925.876675:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784948925.876676:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1784948925.876686:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948925.876694:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948925.876696:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784948925.876701:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948925.876702:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.876709:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948925.876711:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1784948925.876717:0:335217:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1784948925.876717:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1784948925.876722:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:2.0:1784948925.876723:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948925.876730:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:2.0:1784948925.876731:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289662208:0@lo:400: 00000020:00000040:0.0:1784948925.876737:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:2.0:1784948925.876738:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1784948925.876744:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784948925.876747:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:2.0:1784948925.876749:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784948925.876751:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000001:2.0:1784948925.876754:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.876755:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784948925.876760:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784948925.876761:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948925.876765:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000010:2.0:1784948925.876768:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 00000020:00000040:0.0:1784948925.876769:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:2.0:1784948925.876775:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948925.876780:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00080000:0.0:1784948925.876783:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000400:00000010:2.0:1784948925.876785:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005cc7db3b. 00000100:00000001:0.0:1784948925.876786:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948925.876792:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1784948925.876797:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 02000000:00000001:0.0:1784948925.876801:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948925.876806:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948925.876808:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289662208, portal 25 00000100:00000001:0.0:1784948925.876809:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1784948925.876816:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948925.876817:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.876821:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558303872 : -115544151247744 : ffff96e9c7e8b480) 00000100:00000010:0.0:1784948925.876826:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000100:00000040:2.0:1784948925.876830:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784948925.876832:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784948925.876840:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784948925.876844:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.876848:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948925.876850:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948925.876854:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1784948925.876855:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784948925.876857:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948925.876860:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000100:00000200:2.0:1784948925.876865:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289662208, offset 0 02000000:00000010:0.0:1784948925.876866:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000867cea4b. 00000100:00000001:0.0:1784948925.876872:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948925.876873:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:0.0:1784948925.876884:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948925.876894:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948925.876899:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948925.876899:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:0.0:1784948925.876946:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_rcv+-1] req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948925.876947:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438300 00000400:00000200:2.0:1784948925.876962:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7e29 [2] + 0 00000400:00000010:2.0:1784948925.876967:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d30ef19f. 00000100:00000001:0.0:1784948925.876972:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948925.876974:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000005df8d94f 00000100:00000040:0.0:1784948925.876978:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948925.876984:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948925.876989:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000400:00000200:2.0:1784948925.876990:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:0.0:1784948925.876998:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:2.0:1784948925.877002:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948925.877004:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948925.877006:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948925.877009:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1784948925.877011:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005df8d94f 02000000:00000001:0.0:1784948925.877013:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948925.877015:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000005df8d94f. 00000100:00000001:3.0:1784948925.877015:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948925.877022:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:0.0:1784948925.877023:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000eafb2cdc. 00000100:00000001:2.0:1784948925.877025:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784948925.877032:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948925.877032:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784948925.877036:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948925.877041:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000db799909 x1871651289662208 msgsize 224 00000100:00000040:0.0:1784948925.877045:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1784948925.877070:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948925.877077:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784948925.877080:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948925.877084:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948925.877086:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.877087:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948925.877096:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948925.877097:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784948925.877098:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c39699a8] to pc [ptlrpcd_00_02+2] req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948925.877104:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784948925.877109:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 02000000:00000001:3.0:1784948925.877114:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948925.877115:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000100:00000001:2.0:1784948925.877123:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1784948925.877140:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289662336:192.168.202.137@tcp:8: 00000100:00000200:2.0:1784948925.877141:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948925.877147:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948925.877155:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289662208 00000100:00000001:3.0:1784948925.877158:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948925.877162:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948925.877167:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:3.0:1784948925.877167:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784948925.877171:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948925.877172:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948925.877174:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948925.877174:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948925.877178:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948925.877185:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948925.877186:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11202) 02000000:00000001:3.0:1784948925.877186:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948925.877197:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948925.877198:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289662208 02000000:00000010:3.0:1784948925.877198:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000005b888794. 00000100:00000001:2.0:1784948925.877200:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784948925.877203:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948925.877208:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784948925.877209:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948925.877209:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.877210:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948925.877211:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:0.0:1784948925.877216:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000020:00000040:1.0:1784948925.877219:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784948925.877220:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948925.877222:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305865259 00000400:00000010:3.0:1784948925.877224:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000de368614. 00000020:00000001:1.0:1784948925.877227:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948925.877232:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1784948925.877234:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:2.0:1784948925.877234:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000094130822. 00000100:00000001:0.0:1784948925.877236:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948925.877239:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000100:00000001:0.0:1784948925.877246:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948925.877246:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948925.877250:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000200:3.0:1784948925.877250:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289662336, portal 4 00000100:00000001:0.0:1784948925.877252:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1784948925.877254:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:0.0:1784948925.877257:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948925.877259:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948925.877262:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948925.877265:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305865259 00000100:00000001:3.0:1784948925.877265:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000040:0.0:1784948925.877268:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948925.877270:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00000010:2.0:1784948925.877276:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fbfea82a. 00000020:00000010:1.0:1784948925.877283:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f7a7941e. 00000100:00000001:2.0:1784948925.877284:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1784948925.877286:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948925.877293:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305865259 00000020:00000010:1.0:1784948925.877295:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000041b4b183. 00000100:00000010:2.0:1784948925.877304:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b9b6859c. 00000100:00000001:0.0:1784948925.877305:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000010:1.0:1784948925.877308:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:0.0:1784948925.877311:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.877313:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:0.0:1784948925.877319:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948925.877324:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305865259 00000100:00000001:3.0:1784948925.877324:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784948925.877325:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948925.877326:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 02000000:00000001:0.0:1784948925.877335:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948925.877336:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:2.0:1784948925.877336:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000addba683. 00000100:00000040:3.0:1784948925.877337:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948925.877342:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:0.0:1784948925.877343:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289662464:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948925.877345:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948925.877346:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948925.877348:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000100:00000001:2.0:1784948925.877353:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948925.877356:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784948925.877359:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948925.877360:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289662336, offset 0 02000000:00000001:0.0:1784948925.877361:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948925.877364:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.877364:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948925.877365:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948925.877374:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948925.877376:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948925.877380:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 00000400:00000200:3.0:1784948925.877382:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948925.877383:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948925.877386:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948925.877394:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f237c65a. 00000400:00000010:0.0:1784948925.877402:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000001:1.0:1784948925.877406:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784948925.877406:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000200:0.0:1784948925.877409:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289662464, portal 10 00000100:00000001:1.0:1784948925.877412:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948925.877428:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948925.877431:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558302976 : -115544151248640 : ffff96e9c7e8b100) 00000100:00100000:1.0:1784948925.877433:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 736 00000400:00000200:3.0:1784948925.877434:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1784948925.877445:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948925.877445:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:3.0:1784948925.877448:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784948925.877453:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742029392 : -115538967522224 : ffff96eafce21450) 00000400:00000200:3.0:1784948925.877462:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948925.877471:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948925.877471:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000db799909 x1871651289662208/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:586/0 lens 224/0 e 0 to 0 dl 1784948936 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948925.877474:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:0.0:1784948925.877482:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948925.877489:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3414:3414:240:4294967295] 192.168.202.137@tcp LPNI seq info [2200:2200:8:4294967295] 00000400:00000010:0.0:1784948925.877490:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000100:00000200:0.0:1784948925.877498:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289662464, offset 0 00000100:00000001:2.0:1784948925.877508:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948925.877512:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784948925.877513:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948925.877514:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948925.877518:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:1.0:1784948925.877525:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948925.877530:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000400:00000200:3.0:1784948925.877534:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:1.0:1784948925.877540:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000db799909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289662208:12345-0@lo:400: 00000100:00000200:1.0:1784948925.877548:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289662208 00000800:00000200:3.0:1784948925.877554:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784948925.877555:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784948925.877559:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:1.0:1784948925.877563:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784948925.877581:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1784948925.877584:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948925.877586:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000400:00000200:0.0:1784948925.877592:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:1.0:1784948925.877593:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948925.877609:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:0.0:1784948925.877615:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1784948925.877616:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948925.877621:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948925.877622:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784948925.877626:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:1.0:1784948925.877626:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948925.877632:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948925.877637:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948925.877638:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3415:3415:239:4294967295] 192.168.202.137@tcp LPNI seq info [2201:2201:7:4294967295] 00000020:00000001:1.0:1784948925.877643:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000200:3.0:1784948925.877645:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784948925.877648:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948925.877654:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000200:3.0:1784948925.877656:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 616 niov 1 nkiov 1 02000000:00000010:1.0:1784948925.877660:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002afa749d. 02000000:00000001:1.0:1784948925.877667:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948925.877670:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948925.877671:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:1.0:1784948925.877678:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948925.877683:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289662208 00000800:00000200:0.0:1784948925.877685:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948925.877685:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948925.877687:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000010:0.0:1784948925.877695:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b51ad697. 00010000:00000200:1.0:1784948925.877698:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000db799909 x1871651289662208/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:586/0 lens 224/224 e 0 to 0 dl 1784948936 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948925.877698:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000800:00000200:0.0:1784948925.877709:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948925.877709:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305865259 00010000:00000001:1.0:1784948925.877720:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948925.877721:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305865259 00010000:00000001:1.0:1784948925.877725:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948925.877726:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784948925.877731:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305865259 00000800:00000200:0.0:1784948925.877733:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b51ad697 type 1, nob 320 niov 1 nkiov 1 00000100:00001000:1.0:1784948925.877734:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=97 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948925.877739:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305865259 00000100:00000001:1.0:1784948925.877745:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784948925.877747:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305865259 00000100:00000040:1.0:1784948925.877750:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948925.877755:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948925.877757:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784948925.877757:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305865259 00000100:00000040:0.0:1784948925.877762:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2100 usecs (3 jiffies) 02000000:00000001:1.0:1784948925.877764:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784948925.877766:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305865259 02000000:00000001:1.0:1784948925.877768:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.877770:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305865220 00000100:00000001:1.0:1784948925.877776:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948925.877776:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305865259 00000100:00000001:0.0:1784948925.877777:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305865259 00000100:00000040:1.0:1784948925.877785:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948925.877786:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305865220 00000100:00000001:3.0:1784948925.877787:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948925.877791:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305865259 00000400:00000010:1.0:1784948925.877792:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000100:00000001:3.0:1784948925.877795:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.877799:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305865220 00000100:00000200:1.0:1784948925.877800:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289662208, offset 224 00000100:00000001:3.0:1784948925.877803:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.877805:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305865259 00000100:00000001:0.0:1784948925.877811:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305865220 00000100:00000001:3.0:1784948925.877812:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948925.877815:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948925.877817:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305865259 00000100:00000001:0.0:1784948925.877823:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948925.877824:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948925.877825:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948925.877832:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784948925.877832:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1784948925.877833:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.877843:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948925.877844:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948925.877849:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438300 00000100:00000001:3.0:1784948925.877850:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948925.877858:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948925.877861:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x883d [1] + 224 00000800:00000001:0.0:1784948925.877865:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948925.877865:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948925.877869:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1784948925.877872:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1784948925.877878:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000010:2.0:1784948925.877880:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000800:00000001:0.0:1784948925.877884:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948925.877885:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1784948925.877887:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948925.877890:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:1.0:1784948925.877892:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948925.877896:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948925.877900:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948925.877941:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948925.877945:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0e1f1a0 00000400:00000010:2.0:1784948925.877952:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0e1f1a0. 00000100:00000001:2.0:1784948925.877958:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1784948925.877959:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948925.877964:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6438380 00000100:00000200:2.0:1784948925.877970:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000100:0.0:1784948925.877983:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289662336 offset 0 length 520: 4 00000400:00000200:0.0:1784948925.877997:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000076d01bd is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:2.0:1784948925.878000:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784948925.878002:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:1.0:1784948925.878004:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1784948925.878009:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948950 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948925.878016:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000100:00000001:2.0:1784948925.878018:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948925.878019:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784948925.878021:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000100:00000001:2.0:1784948925.878025:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948925.878028:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948925.878031:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948925.878040:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948925.878044:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784948925.878047:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948925.878049:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.878051:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948925.878052:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948925.878056:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948925.878062:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db799909 x1871651289662208/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:586/0 lens 224/224 e 0 to 0 dl 1784948936 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948925.878063:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:0.0:1784948925.878064:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948925.878069:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000001:2.0:1784948925.878073:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948925.878074:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005cc7db3b. 00000800:00000001:2.0:1784948925.878082:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:1.0:1784948925.878083:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000db799909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289662208:12345-0@lo:400: Request processed in 567us (1036us total) trans 0 rc 0/0 00000400:00000200:0.0:1784948925.878085:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000800:00000001:2.0:1784948925.878090:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948925.878091:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000800:00000200:3.0:1784948925.878099:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948925.878101:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784948925.878106:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 736 00000800:00000010:3.0:1784948925.878109:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b51ad697. 00000400:00000200:2.0:1784948925.878110:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:0.0:1784948925.878112:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948925.878115:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948925.878123:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000200:3.0:1784948925.878123:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948925.878128:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:2.0:1784948925.878131:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438400 00000100:00000200:0.0:1784948925.878138:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948925.878139:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000041b4b183. 00000400:00000200:3.0:1784948925.878146:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:1.0:1784948925.878147:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000400:00000200:2.0:1784948925.878148:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 160528 00000020:00000010:1.0:1784948925.878156:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000f7a7941e. 00000800:00000001:2.0:1784948925.878167:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948925.878169:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1784948925.878170:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:3.0:1784948925.878172:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000100:00000001:0.0:1784948925.878173:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948925.878179:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784948925.878180:0:337119:0:(client.c:1580:after_reply()) Process entered 00000800:00000001:2.0:1784948925.878181:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948925.878181:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 02000000:00000010:1.0:1784948925.878183:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000002afa749d. 02000000:00000001:0.0:1784948925.878187:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000200:2.0:1784948925.878189:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948925.878192:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:1.0:1784948925.878194:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948925.878196:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948925.878196:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948925.878199:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948925.878200:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:0.0:1784948925.878204:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948925.878209:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000200:3.0:1784948925.878210:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:0.0:1784948925.878214:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=97 v=5 (5 5 5 5) 00000100:00000001:2.0:1784948925.878214:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000400:00000010:1.0:1784948925.878217:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b62f2953. 00000100:00000200:2.0:1784948925.878218:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00001000:0.0:1784948925.878229:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=97 v=5 (1 1 1 1) 00000400:00000010:1.0:1784948925.878230:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b0e9c277. 00000100:00000010:2.0:1784948925.878230:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086d6bb31. 00000100:00000040:2.0:1784948925.878238:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000086d6bb31 x1871651289662464 msgsize 224 00000100:00000001:0.0:1784948925.878242:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948925.878245:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948925.878246:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948925.878253:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:0.0:1784948925.878254:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000040:3.0:1784948925.878258:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948925.878261:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.878267:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.878272:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:1.0:1784948925.878273:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948925.878277:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948925.878282:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948925.878283:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948925.878292:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948925.878292:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948925.878293:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948925.878297:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948925.878303:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948925.878304:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948925.878309:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948925.878313:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948925.878322:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289662208:0@lo:400: 00000100:00000001:2.0:1784948925.878323:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784948925.878329:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289662464 00000100:00000001:0.0:1784948925.878331:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948925.878332:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948925.878335:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784948925.878336:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948925.878341:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948925.878344:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f59e6b47 x1871651289662208/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948925.878352:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.878354:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1784948925.878359:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289662464 02000000:00000001:0.0:1784948925.878360:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1784948925.878365:0:342763:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000010:0.0:1784948925.878366:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 00000020:00000040:2.0:1784948925.878369:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000001:0.0:1784948925.878373:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948925.878374:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:0.0:1784948925.878379:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948925.878380:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000040:0.0:1784948925.878384:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784948925.878385:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948925.878390:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:2.0:1784948925.878391:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000010:0.0:1784948925.878393:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002aeee8e9. 00000100:00000001:2.0:1784948925.878396:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:0.0:1784948925.878398:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948925.878401:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000010:2.0:1784948925.878402:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000002fef99f7. 00000100:00000010:0.0:1784948925.878403:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f59e6b47. 00000100:00000001:2.0:1784948925.878407:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948925.878410:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000010:2.0:1784948925.878412:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000002fef99f7. 00000100:00000001:0.0:1784948925.878413:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948925.878420:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948925.878422:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006cd4526f. 00000100:00000001:0.0:1784948925.878424:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948925.878428:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948925.878429:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c335e9c8. 00000100:00000001:0.0:1784948925.878431:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948925.878434:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948925.878435:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000040:2.0:1784948925.878444:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948925.878446:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948925.878448:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948925.878451:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948925.878458:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948925.878460:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784948925.878461:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.878475:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948925.878483:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948925.878485:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784948925.878489:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 712 00000100:00000040:2.0:1784948925.878492:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948925.878494:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470019712 : -115544239531904 : ffff96e9c2a59880) 00000100:00000040:2.0:1784948925.878499:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000086d6bb31 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:586/0 lens 224/0 e 0 to 0 dl 1784948936 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948925.878508:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948925.878511:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000002fef99f7. 00000100:00000001:2.0:1784948925.878514:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948925.878516:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000002fef99f7. 00000100:00100000:2.0:1784948925.878520:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000086d6bb31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289662464:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784948925.878524:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289662464 00000020:00000001:2.0:1784948925.878526:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948925.878529:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948925.878530:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948925.878532:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948925.878534:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784948925.878537:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948925.878540:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948925.878542:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948925.878544:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948925.878546:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948925.878548:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948925.878552:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948925.878553:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948925.878556:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000036dde263. 02000000:00000001:2.0:1784948925.878558:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.878560:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948925.878562:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948925.878564:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289662464 00010000:00000001:2.0:1784948925.878566:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948925.878586:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000086d6bb31 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:586/0 lens 224/224 e 0 to 0 dl 1784948936 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948925.878604:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948925.878606:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948925.878609:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=92 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948925.878612:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948925.878614:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948925.878616:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948925.878619:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948925.878620:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948925.878622:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948925.878624:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784948925.878626:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000100:00000200:2.0:1784948925.878628:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289662464, offset 224 00000400:00000200:2.0:1784948925.878633:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948925.878639:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948925.878644:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3416:3416:240:4294967295] 192.168.202.137@tcp LPNI seq info [2202:2202:8:4294967295] 00000400:00000200:2.0:1784948925.878653:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948925.878657:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948925.878661:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000000475ac2. 00000800:00000200:2.0:1784948925.878665:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948925.878670:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784948925.878674:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000000475ac2 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948925.878685:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948925.878688:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948925.878690:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948925.878691:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948925.878693:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948925.878696:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086d6bb31 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:586/0 lens 224/224 e 0 to 0 dl 1784948936 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948925.878705:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000086d6bb31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289662464:12345-192.168.202.137@tcp:400: Request processed in 195us (465us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948925.878714:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 712 00000100:00000040:2.0:1784948925.878717:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948925.878720:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948925.878722:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948925.878724:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c335e9c8. 00000020:00000010:2.0:1784948925.878728:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784948925.878732:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006cd4526f. 00000020:00000040:2.0:1784948925.878735:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948925.878738:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948925.878768:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948925.878772:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000000475ac2. 00000800:00000001:2.0:1784948925.878777:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948925.878779:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948925.878786:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948925.878787:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948925.878791:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948925.878794:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000800:00000001:2.0:1784948925.878796:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948925.878799:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000100:00000001:1.0:1784948925.878807:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:2.0:1784948925.878807:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784948925.878812:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948925.878819:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438400 00000400:00000200:2.0:1784948925.878825:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x884d [1] + 224 00000800:00000001:2.0:1784948925.878830:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948925.878835:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948925.878838:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948925.878841:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948925.878845:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948925.878849:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948925.878857:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948925.878872:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948925.878877:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948925.878879:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948925.878962:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948925.878966:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948925.878972:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948925.878973:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948925.878976:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f237c65a. 00000400:00000200:0.0:1784948925.878981:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:0.0:1784948925.878983:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784948925.878986:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948925.878991:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948925.879001:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948925.879008:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948925.879010:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948925.879012:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948925.879014:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948925.879015:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948925.879016:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948925.879018:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948925.879023:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=92 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948925.879025:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=92 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948925.879028:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948925.879029:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948925.879031:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948925.879033:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948925.879041:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.879044:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948925.879046:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948925.879047:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948925.879048:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948925.879051:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948925.879058:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948925.879060:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948925.879061:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948925.879064:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948925.879072:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289662464:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948925.879076:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948925.879077:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948925.879079:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c39699a8 x1871651289662464/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948925.879084:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948925.879085:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948925.879087:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 02000000:00000001:0.0:1784948925.879090:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948925.879091:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948925.879092:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948925.879094:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948925.879095:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000eafb2cdc. 02000000:00000001:0.0:1784948925.879097:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948925.879098:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948925.879099:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c39699a8. 00000100:00000001:0.0:1784948925.879101:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948925.879103:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948925.879104:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948925.879108:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948925.879109:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948925.879111:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948925.879113:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948925.919996:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305865259, at 4305865260 00000100:00000040:1.0:1784948925.920010:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305865259, at 4305865260 00000100:00000010:3.0:1784948925.920012:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000addba683. 00000100:00000040:0.0:1784948925.920017:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305865259, at 4305865260 00000100:00000010:1.0:1784948925.920028:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fbfea82a. 00000100:00000010:0.0:1784948925.920031:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000094130822. 00000100:00000040:2.0:1784948925.920034:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305865259, at 4305865260 00000100:00000010:2.0:1784948925.920049:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b9b6859c. 00010000:00000001:2.0:1784948926.708065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948926.708079:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948926.708089:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948926.708091:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948926.708241:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948926.708255:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948926.708259:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948927.731993:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948927.732003:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948927.732011:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948927.732013:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948927.732101:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948927.732106:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948927.732110:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948928.755989:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948928.755998:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948928.756005:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948928.756006:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948928.756096:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948928.756101:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948928.756104:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948929.780041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948929.780054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948929.780065:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948929.780067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948929.780186:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948929.780193:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948929.780198:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948930.804013:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948930.804025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948930.804033:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948930.804037:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948930.804041:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948930.804045:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948930.804048:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948930.804051:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948930.804053:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948930.804055:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948930.804163:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948930.804169:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948930.804173:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948930.996030:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948930.996050:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948930.996056:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948930.996064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948930.996069:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948930.996091:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948930.996096:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.996180:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948930.996191:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948930.996202:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000053d87417. 00000020:00000040:1.0:1784948930.996212:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948930.996221:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948930.996226:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948930.996237:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948930.996249:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:1.0:1784948930.996262:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.996283:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948930.996307:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948930.996312:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.996325:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053d87417] to pc [ptlrpcd_00_03+3] req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.996362:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948930.996366:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948930.996371:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948930.996376:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948930.996382:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948930.996386:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006f266164. 00000020:00000040:1.0:1784948930.996389:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948930.996393:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948930.996396:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948930.996398:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948930.996409:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000061dbde97. 00000100:00000001:1.0:1784948930.996414:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.996424:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948930.996432:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948930.996436:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.996445:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006f266164] to pc [ptlrpcd_00_00+0] req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.996490:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.996498:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11208) 00000100:00000001:0.0:1784948930.996503:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948930.996520:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948930.996522:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948930.996557:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1784948930.996558:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948930.996574:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948930.996587:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784948930.996589:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.996592:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948930.996605:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948930.996605:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948930.996613:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948930.996615:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.996621:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948930.996624:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948930.996630:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.996632:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948930.996646:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948930.996658:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1784948930.996659:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948930.996662:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948930.996673:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948930.996675:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948930.996687:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784948930.996694:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000053d87417 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289662592:0@lo:400: 02000000:00000001:1.0:1784948930.996698:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948930.996712:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1784948930.996713:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006f266164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289662720:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948930.996722:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948930.996724:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948930.996728:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948930.996729:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948930.996737:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948930.996738:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1784948930.996750:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948930.996754:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 02000000:00000010:1.0:1784948930.996757:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 02000000:00000001:0.0:1784948930.996768:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948930.996770:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948930.996771:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948930.996785:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948930.996796:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f237c65a. 00000400:00000010:1.0:1784948930.996798:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009a55be35. 00000400:00000010:1.0:1784948930.996822:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000010:0.0:1784948930.996828:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000200:1.0:1784948930.996837:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289662720, portal 10 00000100:00000200:0.0:1784948930.996841:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289662592, portal 25 00000100:00000001:1.0:1784948930.996843:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948930.996852:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549160704 : -115544160390912 : ffff96e9c75d3100) 00000100:00000001:0.0:1784948930.996854:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948930.996866:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549159808 : -115544160391808 : ffff96e9c75d2d80) 00000100:00000040:1.0:1784948930.996874:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948930.996883:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.996953:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948930.996967:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948930.996982:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000100:00000001:0.0:1784948930.996984:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:1.0:1784948930.996992:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289662720, offset 0 00000100:00000040:0.0:1784948930.997000:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784948930.997007:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000400:00000200:1.0:1784948930.997016:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000200:0.0:1784948930.997025:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289662592, offset 0 00000400:00000200:0.0:1784948930.997043:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948930.997048:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948930.997058:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948930.997066:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948930.997072:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948930.997077:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948930.997085:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3417:3417:240:4294967295] 192.168.202.137@tcp LPNI seq info [2203:2203:8:4294967295] 00000400:00000200:1.0:1784948930.997101:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948930.997109:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948930.997118:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000400:00000200:0.0:1784948930.997123:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:1.0:1784948930.997140:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784948930.997148:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438480 00000400:00000200:0.0:1784948930.997174:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7e45 [2] + 0 00000800:00000200:1.0:1784948930.997175:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948930.997194:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:0.0:1784948930.997195:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e6551ee. 00000400:00000200:0.0:1784948930.997214:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b1caac62 00000400:00000200:0.0:1784948930.997225:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948930.997225:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.997243:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2100 usecs (3 jiffies) 00000400:00000200:0.0:1784948930.997249:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948930.997253:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305870340 00000400:00000200:0.0:1784948930.997258:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b1caac62 00000100:00000010:1.0:1784948930.997265:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000400:00000010:0.0:1784948930.997266:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b1caac62. 00000100:00000001:0.0:1784948930.997290:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948930.997291:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000200:0.0:1784948930.997299:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784948930.997305:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000001a09b990 x1871651289662592 msgsize 224 00000100:00000001:1.0:1784948930.997307:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305870340 00000100:00000010:1.0:1784948930.997322:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00100000:0.0:1784948930.997324:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948930.997344:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948930.997346:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948930.997358:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305870340 00000100:00000010:1.0:1784948930.997364:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:0.0:1784948930.997366:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784948930.997379:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000400:00000200:0.0:1784948930.997385:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000100:00000001:1.0:1784948930.997387:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305870340 00000400:00000010:0.0:1784948930.997392:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000100:00000010:1.0:1784948930.997397:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:0.0:1784948930.997400:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948930.997404:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000200:0.0:1784948930.997411:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.997413:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948930.997430:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948930.997437:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948930.997448:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784948930.997454:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000100:00000001:1.0:1784948930.997466:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948930.997480:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948930.997481:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784948930.997492:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948930.997498:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948930.997511:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948930.997518:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948930.997551:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000100:00000001:0.0:1784948930.997553:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784948930.997556:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948930.997561:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000100:00000001:0.0:1784948930.997575:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948930.997580:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948930.997584:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305870379 00000800:00000001:1.0:1784948930.997590:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948930.997593:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948930.997602:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305870340 00000400:00000200:1.0:1784948930.997613:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784948930.997615:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948930.997620:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305870379 00000100:00000001:0.0:1784948930.997643:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305870340 00000100:00000001:2.0:1784948930.997647:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948930.997648:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438500 00000100:00000001:0.0:1784948930.997668:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305870379 00000100:00000040:2.0:1784948930.997670:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948930.997683:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 160752 00000100:00000001:0.0:1784948930.997697:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305870340 00000800:00000001:1.0:1784948930.997711:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948930.997727:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948930.997729:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305870379 00000800:00000001:1.0:1784948930.997734:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948930.997745:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784948930.997751:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948930.997755:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305870340 00000400:00000200:1.0:1784948930.997772:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948930.997777:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948930.997783:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.997784:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948930.997793:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784948930.997795:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948930.997806:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000acf1d731. 00000100:00000001:0.0:1784948930.997808:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948930.997821:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:1.0:1784948930.997829:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000acf1d731 x1871651289662720 msgsize 224 00000100:00100000:2.0:1784948930.997831:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289662592 00000100:00000001:0.0:1784948930.997838:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1784948930.997840:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948930.997850:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948930.997851:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:1.0:1784948930.997862:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784948930.997862:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948930.997866:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948930.997879:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1784948930.997882:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948930.997889:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948930.997892:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289662592 00000100:00000001:1.0:1784948930.997894:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948930.997899:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948930.997901:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948930.997908:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000001:1.0:1784948930.997909:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948930.997959:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948930.997964:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948930.997973:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784948930.997980:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784948930.997986:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948930.997994:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948930.998002:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:1.0:1784948930.998006:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948930.998007:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1784948930.998014:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289662720 00000020:00000010:2.0:1784948930.998015:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 02000000:00000001:1.0:1784948930.998021:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948930.998025:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948930.998028:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948930.998029:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006cd4526f. 02000000:00000001:1.0:1784948930.998033:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948930.998039:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000016271b20. 00000100:00100000:1.0:1784948930.998040:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289662720 00000020:00000001:1.0:1784948930.998046:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948930.998050:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000010:2.0:1784948930.998052:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000020:00000001:1.0:1784948930.998054:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948930.998062:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948930.998066:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948930.998070:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:2.0:1784948930.998071:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948930.998075:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948930.998079:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000010:1.0:1784948930.998083:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:2.0:1784948930.998084:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948930.998090:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.998091:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948930.998099:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00000001:2.0:1784948930.998108:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948930.998110:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f047fe3. 00000020:00000010:1.0:1784948930.998121:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000041b4b183. 00000020:00000010:1.0:1784948930.998127:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948930.998137:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948930.998145:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948930.998151:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948930.998153:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948930.998156:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948930.998166:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948930.998167:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948930.998176:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948930.998179:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948930.998186:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948930.998194:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 737 00000100:00000001:3.0:1784948930.998194:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948930.998203:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:2.0:1784948930.998205:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948930.998208:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948930.998211:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948930.998213:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742034512 : -115538967517104 : ffff96eafce22850) 00000100:00000001:3.0:1784948930.998221:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948930.998230:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001a09b990 x1871651289662592/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:591/0 lens 224/0 e 0 to 0 dl 1784948941 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.998233:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948930.998239:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948930.998251:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:1.0:1784948930.998257:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 713 00000020:00000010:2.0:1784948930.998258:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000040:1.0:1784948930.998269:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948930.998272:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948930.998276:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549162496 : -115544160389120 : ffff96e9c75d3800) 00000020:00000010:2.0:1784948930.998278:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000040:1.0:1784948930.998293:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000acf1d731 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:591/0 lens 224/0 e 0 to 0 dl 1784948941 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948930.998301:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001a09b990 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289662592:12345-0@lo:400: 00000100:00000200:2.0:1784948930.998319:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289662592 00000100:00000001:1.0:1784948930.998322:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:2.0:1784948930.998325:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000010:1.0:1784948930.998328:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000020:00000001:2.0:1784948930.998332:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784948930.998338:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1784948930.998339:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948930.998347:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000020:00000001:2.0:1784948930.998351:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948930.998357:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948930.998364:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00100000:1.0:1784948930.998365:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000acf1d731 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289662720:12345-192.168.202.137@tcp:400: 00000020:00000001:2.0:1784948930.998371:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948930.998377:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289662720 00000020:00000001:2.0:1784948930.998379:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948930.998384:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948930.998385:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948930.998393:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948930.998394:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948930.998399:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000001:1.0:1784948930.998400:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948930.998406:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:2.0:1784948930.998407:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:1.0:1784948930.998409:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 02000000:00000001:2.0:1784948930.998415:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000001:1.0:1784948930.998416:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948930.998423:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784948930.998425:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000de31af8d. 00000020:00000001:1.0:1784948930.998426:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948930.998431:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:2.0:1784948930.998433:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948930.998435:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948930.998439:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948930.998440:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000001:2.0:1784948930.998445:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.998449:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00010000:00000040:2.0:1784948930.998451:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289662592 02000000:00000001:1.0:1784948930.998453:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00010000:00000001:2.0:1784948930.998458:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000010:1.0:1784948930.998462:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b0c59b90. 00010000:00000200:2.0:1784948930.998468:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001a09b990 x1871651289662592/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:591/0 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948930.998469:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.998476:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948930.998482:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948930.998488:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289662720 00010000:00000001:1.0:1784948930.998495:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948930.998502:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000acf1d731 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:591/0 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948930.998505:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948930.998512:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948930.998516:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948930.998521:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948930.998523:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=102 v=5 (1 1 1 1) 00000100:00001000:1.0:1784948930.998545:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=97 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948930.998562:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948930.998562:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948930.998569:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948930.998575:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000040:2.0:1784948930.998575:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 02000000:00000001:1.0:1784948930.998583:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784948930.998587:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948930.998588:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.998597:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948930.998599:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948930.998604:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.998605:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948930.998613:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784948930.998615:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000100:00000040:2.0:1784948930.998622:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000200:1.0:1784948930.998624:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289662720, offset 224 00000400:00000010:2.0:1784948930.998630:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:1.0:1784948930.998639:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948930.998649:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000200:2.0:1784948930.998650:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289662592, offset 224 00000400:00000200:1.0:1784948930.998659:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3418:3418:240:4294967295] 192.168.202.137@tcp LPNI seq info [2204:2204:8:4294967295] 00000400:00000200:2.0:1784948930.998669:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948930.998679:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948930.998686:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948930.998692:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000400:00000200:2.0:1784948930.998697:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:1.0:1784948930.998704:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948930.998716:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000200:2.0:1784948930.998718:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438480 00000800:00000200:1.0:1784948930.998723:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1784948930.998732:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8865 [1] + 224 00000400:00000200:2.0:1784948930.998742:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948930.998747:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948930.998748:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948930.998753:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784948930.998759:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000200:2.0:1784948930.998764:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948930.998766:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948930.998771:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948930.998778:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948930.998787:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000acf1d731 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:591/0 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948930.998792:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948930.998826:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000acf1d731 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289662720:12345-192.168.202.137@tcp:400: Request processed in 498us (991us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948930.998835:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 713 00000100:00000040:1.0:1784948930.998840:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948930.998840:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000200:3.0:1784948930.998845:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948930.998848:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000010:3.0:1784948930.998853:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000400:00000200:2.0:1784948930.998854:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000100:00000001:1.0:1784948930.998855:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000010:2.0:1784948930.998863:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000020:00000010:1.0:1784948930.998865:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000041b4b183. 00000400:00000200:3.0:1784948930.998866:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948930.998869:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000010:1.0:1784948930.998877:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948930.998878:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948930.998885:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948930.998887:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948930.998889:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f047fe3. 00000400:00000200:3.0:1784948930.998895:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000100:00000040:2.0:1784948930.998898:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000040:1.0:1784948930.998900:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000400:00000010:3.0:1784948930.998901:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00010000:00000001:2.0:1784948930.998949:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948930.998954:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.998958:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948930.998960:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784948930.998965:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948930.998972:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948930.998978:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948930.998978:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001a09b990 x1871651289662592/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:591/0 lens 224/224 e 0 to 0 dl 1784948941 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948930.998983:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948930.998996:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:3.0:1784948930.999001:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948930.999011:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:0.0:1784948930.999014:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948930.999021:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000001:3.0:1784948930.999023:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948930.999024:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001a09b990 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289662592:12345-0@lo:400: Request processed in 767us (1711us total) trans 0 rc 0/0 00000400:00000010:0.0:1784948930.999032:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f237c65a. 00000400:00000200:0.0:1784948930.999052:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000200:3.0:1784948930.999053:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:0.0:1784948930.999068:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00100000:2.0:1784948930.999068:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 737 00000400:00000200:3.0:1784948930.999087:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438500 00000100:00000001:0.0:1784948930.999092:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1784948930.999094:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:3.0:1784948930.999103:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8861 [1] + 224 00000100:00000001:2.0:1784948930.999105:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:0.0:1784948930.999115:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948930.999117:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:3.0:1784948930.999123:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784948930.999128:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000016271b20. 00000800:00000001:3.0:1784948930.999136:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948930.999145:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000400:00000200:3.0:1784948930.999152:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:2.0:1784948930.999160:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006cd4526f. 00000020:00000040:2.0:1784948930.999177:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:3.0:1784948930.999183:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:2.0:1784948930.999205:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948930.999206:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948930.999208:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784948930.999212:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000de31af8d. 00000100:00000200:3.0:1784948930.999223:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948930.999229:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784948930.999236:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948930.999255:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:2.0:1784948930.999273:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a86c77e2. 00000400:00000010:2.0:1784948930.999288:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000995a12aa. 00000100:00000001:0.0:1784948930.999310:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948930.999317:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948930.999322:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948930.999324:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948930.999332:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948930.999340:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948930.999347:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948930.999352:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948930.999360:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00001000:0.0:1784948930.999361:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=102 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948930.999368:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=102 v=5 (1 1 1 1) 00000800:00000001:3.0:1784948930.999370:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948930.999373:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948930.999376:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948930.999377:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948930.999383:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948930.999387:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948930.999392:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948930.999398:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948930.999407:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948930.999411:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948930.999414:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948930.999418:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948930.999427:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000053d87417 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289662592:0@lo:400: 00000100:00000001:0.0:1784948930.999436:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948930.999440:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948930.999445:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000053d87417 x1871651289662592/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948930.999453:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948930.999457:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948930.999462:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 00000100:00000001:1.0:1784948930.999466:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948930.999468:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1784948930.999474:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784948930.999476:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948930.999488:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000001:1.0:1784948930.999495:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1784948930.999497:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000001:1.0:1784948930.999509:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000010:0.0:1784948930.999511:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 00000400:00000010:1.0:1784948930.999518:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009a55be35. 02000000:00000001:0.0:1784948930.999522:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948930.999553:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948930.999556:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000053d87417. 00000400:00000200:1.0:1784948930.999573:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000100:00000001:0.0:1784948930.999574:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948930.999585:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948930.999585:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000100:00000001:0.0:1784948930.999605:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948930.999607:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948930.999614:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948930.999621:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948930.999621:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948930.999640:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948930.999650:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948930.999658:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.999669:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948930.999671:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948930.999675:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948930.999679:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948930.999684:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948930.999686:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948930.999689:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948930.999696:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=97 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948930.999700:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=97 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948930.999703:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948930.999705:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948930.999707:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948930.999712:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948930.999719:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.999723:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948930.999725:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948930.999727:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948930.999728:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948930.999731:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.999739:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948930.999742:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948930.999744:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948930.999747:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948930.999754:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006f266164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289662720:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948930.999759:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948930.999761:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948930.999764:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006f266164 x1871651289662720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948946 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948930.999770:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948930.999773:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948930.999775:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 02000000:00000001:1.0:1784948930.999782:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948930.999784:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948930.999786:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948930.999789:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948930.999791:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000061dbde97. 02000000:00000001:1.0:1784948930.999795:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948930.999797:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948930.999799:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006f266164. 00000100:00000001:1.0:1784948930.999802:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948930.999804:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948930.999806:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948930.999811:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948930.999812:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948930.999814:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948930.999817:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948931.040046:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305870379, at 4305870380 00000100:00000010:3.0:1784948931.040062:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:0.0:1784948931.040063:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305870379, at 4305870380 00000100:00000010:0.0:1784948931.040082:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:1.0:1784948931.040085:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305870379, at 4305870380 00000100:00000040:2.0:1784948931.040097:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305870379, at 4305870380 00000100:00000010:1.0:1784948931.040101:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:2.0:1784948931.040119:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784948931.828055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948931.828066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948931.828075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948931.828077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948931.828222:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948931.828231:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948931.828237:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948932.020031:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948932.020064:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948932.020067:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948932.020073:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948932.020076:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948932.020082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948932.020086:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948932.851989:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948932.851999:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948932.852006:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948932.852007:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948932.852081:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948932.852085:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948932.852089:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948933.043980:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948933.043995:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948933.043997:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948933.044002:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948933.044006:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948933.044010:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948933.044012:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948933.876011:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948933.876023:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948933.876031:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948933.876033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948933.876139:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948933.876149:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948933.876157:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948934.068034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948934.068053:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948934.068056:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948934.068062:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948934.068066:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948934.068072:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948934.068076:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948934.900012:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948934.900025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948934.900034:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948934.900036:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948934.900157:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948934.900164:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948934.900169:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948935.092051:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948935.092069:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948935.092072:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948935.092079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948935.092083:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948935.092089:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948935.092092:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948935.924070:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948935.924083:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948935.924094:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948935.924096:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948935.924190:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948935.924198:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948935.924204:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948936.116154:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1784948936.116155:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948936.116169:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948936.116174:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:3.0:1784948936.116181:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000074c559e1. 00000100:00000001:1.0:1784948936.116182:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784948936.116209:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948936.116210:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948936.116219:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948936.116222:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784948936.116225:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948936.116227:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948936.116240:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948936.116245:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948936.116258:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:3.0:1784948936.116269:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948936.116287:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948936.116304:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948936.116308:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948936.116315:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000074c559e1] to pc [ptlrpcd_00_01+1] req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948936.116361:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784948936.116369:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784948936.116374:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784948936.116382:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948936.116390:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784948936.116397:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000020:00000040:3.0:1784948936.116402:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948936.116403:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948936.116408:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948936.116411:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784948936.116413:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948936.116418:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948936.116421:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784948936.116423:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1784948936.116433:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948936.116444:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000040:1.0:1784948936.116446:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948936.116460:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948936.116517:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948936.116525:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948936.116532:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948936.116536:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948936.116540:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948936.116542:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948936.116549:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948936.116555:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_00_02+2] req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948936.116562:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948936.116586:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289662848:0@lo:400: 00000100:00000001:1.0:1784948936.116608:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948936.116615:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948936.116621:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784948936.116626:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11213) 02000000:00000001:1.0:1784948936.116628:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948936.116650:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948936.116651:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784948936.116668:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 02000000:00000010:1.0:1784948936.116671:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 00000100:00000001:0.0:1784948936.116676:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948936.116679:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.116691:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:1.0:1784948936.116697:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009a55be35. 00000100:00000001:0.0:1784948936.116699:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948936.116716:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948936.116721:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:1.0:1784948936.116722:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000100:00000001:0.0:1784948936.116725:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000200:1.0:1784948936.116747:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289662848, portal 25 00000100:00000040:0.0:1784948936.116750:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948936.116762:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948936.116768:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554956160 : -115544154595456 : ffff96e9c7b59f80) 00000100:00000040:1.0:1784948936.116789:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948936.116790:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948936.116791:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948936.116802:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.116805:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948936.116833:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948936.116835:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948936.116846:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784948936.116848:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948936.116854:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 02000000:00000001:0.0:1784948936.116865:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948936.116865:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289662848, offset 0 00000100:00100000:0.0:1784948936.116898:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289662976:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784948936.116902:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948936.116964:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948936.116979:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948936.116985:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948936.116992:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784948936.116999:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:0.0:1784948936.117000:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 02000000:00000001:0.0:1784948936.117012:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948936.117022:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f237c65a. 00000400:00000200:1.0:1784948936.117028:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438580 00000400:00000010:0.0:1784948936.117031:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000400:00000200:1.0:1784948936.117044:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7e65 [2] + 0 00000400:00000010:1.0:1784948936.117050:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001db3b9dd. 00000400:00000200:1.0:1784948936.117060:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000034386f9e 00000100:00000200:0.0:1784948936.117070:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289662976, portal 10 00000400:00000200:1.0:1784948936.117073:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948936.117079:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948936.117082:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000400:00000200:1.0:1784948936.117087:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948936.117107:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000034386f9e 00000100:00000040:0.0:1784948936.117113:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948936.117115:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000034386f9e. 00000100:00000001:1.0:1784948936.117131:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948936.117140:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784948936.117142:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948936.117150:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000001eb61cc5 x1871651289662848 msgsize 224 00000100:00000040:0.0:1784948936.117162:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784948936.117174:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000010:0.0:1784948936.117176:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000100:00000040:1.0:1784948936.117178:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000200:0.0:1784948936.117187:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289662976, offset 0 00000400:00000200:0.0:1784948936.117202:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948936.117206:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948936.117226:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:1.0:1784948936.117232:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000100:00000001:1.0:1784948936.117243:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784948936.117250:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000200:1.0:1784948936.117259:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948936.117273:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784948936.117276:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948936.117285:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1784948936.117295:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289662848 00000400:00000200:0.0:1784948936.117304:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:1.0:1784948936.117307:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948936.117308:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948936.117319:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:0.0:1784948936.117329:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784948936.117331:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948936.117349:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:2.0:1784948936.117350:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948936.117352:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948936.117369:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784948936.117371:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289662848 00000400:00000200:0.0:1784948936.117374:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3419:3419:240:4294967295] 192.168.202.137@tcp LPNI seq info [2205:2205:8:4294967295] 00000020:00000001:2.0:1784948936.117382:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948936.117387:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784948936.117394:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948936.117404:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:0.0:1784948936.117411:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:2.0:1784948936.117413:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784948936.117428:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948936.117434:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948936.117438:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948936.117450:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305875499 00000100:00000001:2.0:1784948936.117451:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000010:0.0:1784948936.117454:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000eafb2cdc. 00000100:00000010:1.0:1784948936.117464:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000020:00000010:2.0:1784948936.117466:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000800:00000200:0.0:1784948936.117479:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948936.117505:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784948936.117509:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948936.117524:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00000001:1.0:1784948936.117526:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305875499 00000800:00000200:0.0:1784948936.117540:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000010:1.0:1784948936.117545:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000800:00000200:0.0:1784948936.117548:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000eafb2cdc type 1, nob 320 niov 1 nkiov 1 00000020:00000010:2.0:1784948936.117549:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006cd4526f. 00000100:00000001:1.0:1784948936.117563:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948936.117573:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305875499 00000020:00000010:2.0:1784948936.117573:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000016271b20. 00000100:00000010:1.0:1784948936.117586:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000010:2.0:1784948936.117587:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:0.0:1784948936.117591:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948936.117594:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000040:0.0:1784948936.117614:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2102 usecs (3 jiffies) 00000100:00000040:2.0:1784948936.117615:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948936.117619:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305875499 00000100:00000001:0.0:1784948936.117638:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305875460 00000100:00000010:1.0:1784948936.117641:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:2.0:1784948936.117641:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948936.117651:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948936.117661:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.117663:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305875499 00000100:00000001:1.0:1784948936.117665:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784948936.117675:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305875460 00000100:00000001:1.0:1784948936.117679:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948936.117681:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.117685:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305875499 00000100:00000001:1.0:1784948936.117697:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.117699:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305875460 00000100:00000001:2.0:1784948936.117711:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948936.117718:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305875499 00000100:00000001:1.0:1784948936.117720:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.117740:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305875460 00000100:00000001:1.0:1784948936.117742:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948936.117745:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948936.117755:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948936.117763:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305875499 00000100:00000001:1.0:1784948936.117768:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948936.117771:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948936.117775:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1784948936.117781:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 738 00000100:00000001:0.0:1784948936.117800:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948936.117801:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948936.117802:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948936.117810:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948936.117821:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741505616 : -115539968046000 : ffff96eac13f4c50) 00000100:00000001:1.0:1784948936.117822:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.117826:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948936.117833:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948936.117835:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001eb61cc5 x1871651289662848/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:597/0 lens 224/0 e 0 to 0 dl 1784948947 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.117841:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948936.117867:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948936.117872:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:2.0:1784948936.117876:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948936.117879:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00100000:2.0:1784948936.117887:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001eb61cc5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289662848:12345-0@lo:400: 00000100:00000200:2.0:1784948936.117893:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289662848 00000020:00000001:2.0:1784948936.117899:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948936.117961:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948936.117965:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948936.117971:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948936.117976:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948936.117981:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948936.117985:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948936.117989:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948936.117993:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948936.117998:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948936.118002:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948936.118007:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948936.118014:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000200:3.0:1784948936.118017:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000010:2.0:1784948936.118020:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a6fbdd8b. 00000800:00000001:1.0:1784948936.118029:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784948936.118032:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948936.118032:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000eafb2cdc. 00000100:00000001:2.0:1784948936.118051:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948936.118054:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948936.118061:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948936.118077:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784948936.118078:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948936.118088:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:2.0:1784948936.118092:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289662848 00000400:00000200:3.0:1784948936.118094:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000001:2.0:1784948936.118107:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:3.0:1784948936.118119:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000200:1.0:1784948936.118120:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000200:2.0:1784948936.118121:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001eb61cc5 x1871651289662848/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:597/0 lens 224/224 e 0 to 0 dl 1784948947 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948936.118137:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000100:00000001:3.0:1784948936.118153:0:337112:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:2.0:1784948936.118161:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:1.0:1784948936.118165:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438600 00010000:00000001:2.0:1784948936.118167:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948936.118173:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:2.0:1784948936.118185:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=108 v=5 (1 1 1 1) 00000400:00000200:1.0:1784948936.118199:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 160976 00000100:00000001:2.0:1784948936.118215:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948936.118231:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:1.0:1784948936.118232:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948936.118245:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948936.118250:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:1.0:1784948936.118256:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948936.118258:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1784948936.118260:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:1.0:1784948936.118276:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:2.0:1784948936.118277:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948936.118287:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948936.118289:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948936.118296:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948936.118308:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:2.0:1784948936.118313:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948936.118325:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000010:1.0:1784948936.118330:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006f266164. 00000100:00000200:2.0:1784948936.118340:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289662848, offset 224 00000100:00000040:1.0:1784948936.118345:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000006f266164 x1871651289662976 msgsize 224 00000100:00000001:3.0:1784948936.118373:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948936.118389:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948936.118391:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1784948936.118401:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784948936.118413:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784948936.118425:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784948936.118428:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438580 00000800:00000001:1.0:1784948936.118441:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948936.118441:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x887d [1] + 224 00000800:00000001:1.0:1784948936.118449:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948936.118457:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948936.118468:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948936.118477:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948936.118508:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948936.118547:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.118578:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948936.118579:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:0.0:1784948936.118593:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289662976 00000400:00000200:2.0:1784948936.118598:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 02000000:00000001:0.0:1784948936.118605:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000010:2.0:1784948936.118606:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:0.0:1784948936.118615:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948936.118616:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948936.118623:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948936.118631:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948936.118632:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948936.118632:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948936.118648:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948936.118649:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1784948936.118652:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948936.118655:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1784948936.118657:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000010:1.0:1784948936.118668:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009a55be35. 00010000:00000001:2.0:1784948936.118669:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00100000:0.0:1784948936.118670:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289662976 00000020:00000001:2.0:1784948936.118675:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948936.118682:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000020:00000001:2.0:1784948936.118684:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:0.0:1784948936.118687:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:1.0:1784948936.118691:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000100:00000040:2.0:1784948936.118696:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001eb61cc5 x1871651289662848/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:597/0 lens 224/224 e 0 to 0 dl 1784948947 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784948936.118698:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:1.0:1784948936.118700:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948936.118710:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948936.118713:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948936.118726:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784948936.118747:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00100000:2.0:1784948936.118768:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001eb61cc5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289662848:12345-0@lo:400: Request processed in 895us (1611us total) trans 0 rc 0/0 00000020:00000001:0.0:1784948936.118770:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784948936.118775:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.118798:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948936.118803:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:2.0:1784948936.118808:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 738 00000100:00000001:1.0:1784948936.118809:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948936.118813:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000100:00000040:2.0:1784948936.118815:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948936.118817:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948936.118824:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948936.118825:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948936.118829:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948936.118829:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948936.118831:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:1.0:1784948936.118838:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948936.118841:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000016271b20. 02000000:00000001:1.0:1784948936.118851:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948936.118852:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000020:00000010:2.0:1784948936.118854:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00001000:1.0:1784948936.118862:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=108 v=5 (5 5 5 5) 00000020:00000010:2.0:1784948936.118872:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006cd4526f. 00000100:00001000:1.0:1784948936.118874:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=108 v=5 (1 1 1 1) 00000020:00000010:0.0:1784948936.118878:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:1.0:1784948936.118884:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000040:2.0:1784948936.118885:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784948936.118891:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948936.118895:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000005ae08312. 02000000:00000001:2.0:1784948936.118896:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:1.0:1784948936.118897:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:0.0:1784948936.118914:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00010000:00000001:1.0:1784948936.118942:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948936.118947:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a6fbdd8b. 00000100:00000001:1.0:1784948936.118948:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948936.118957:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000040:1.0:1784948936.118961:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948936.118962:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948936.118977:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784948936.118979:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948936.118987:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:2.0:1784948936.118988:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000093b2e4e6. 00000100:00000001:1.0:1784948936.118993:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948936.118994:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:2.0:1784948936.118997:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005456a5d6. 00000100:00000040:1.0:1784948936.119008:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.119013:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948936.119029:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:1.0:1784948936.119031:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289662848:0@lo:400: 00000100:00000001:0.0:1784948936.119044:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948936.119047:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948936.119057:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948936.119063:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000074c559e1 x1871651289662848/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.119069:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948936.119072:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784948936.119084:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:1.0:1784948936.119085:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948936.119098:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 00000100:00000001:0.0:1784948936.119099:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948936.119112:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948936.119116:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948936.119118:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784948936.119121:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948936.119124:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:1.0:1784948936.119129:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948936.119131:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948936.119134:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000074c559e1. 00000100:00000001:0.0:1784948936.119137:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948936.119139:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948936.119142:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948936.119151:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948936.119156:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948936.119159:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948936.119162:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948936.119166:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.119177:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948936.119183:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948936.119205:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 714 00000100:00000001:2.0:1784948936.119205:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948936.119214:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784948936.119230:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784948936.119234:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.119239:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549160704 : -115544160390912 : ffff96e9c75d3100) 00000100:00000001:2.0:1784948936.119240:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948936.119259:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006f266164 x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:597/0 lens 224/0 e 0 to 0 dl 1784948947 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.119282:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948936.119289:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000100:00000001:0.0:1784948936.119293:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948936.119297:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000100:00100000:0.0:1784948936.119306:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006f266164 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289662976:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948936.119314:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289662976 00000020:00000001:0.0:1784948936.119319:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948936.119325:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948936.119329:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948936.119333:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948936.119337:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948936.119343:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948936.119347:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948936.119351:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948936.119355:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948936.119359:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948936.119364:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948936.119369:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948936.119373:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948936.119380:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000dedc3ae5. 02000000:00000001:0.0:1784948936.119384:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.119388:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948936.119392:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948936.119396:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289662976 00010000:00000001:0.0:1784948936.119401:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948936.119407:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006f266164 x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:597/0 lens 224/224 e 0 to 0 dl 1784948947 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948936.119421:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948936.119426:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948936.119431:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=103 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948936.119438:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948936.119442:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948936.119448:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948936.119452:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948936.119455:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.119461:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948936.119466:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948936.119472:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000100:00000200:0.0:1784948936.119480:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289662976, offset 224 00000400:00000200:0.0:1784948936.119519:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948936.119536:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948936.119546:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3420:3420:240:4294967295] 192.168.202.137@tcp LPNI seq info [2206:2206:8:4294967295] 00000400:00000200:0.0:1784948936.119564:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948936.119574:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948936.119581:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002aeee8e9. 00000800:00000200:0.0:1784948936.119591:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948936.119600:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948936.119606:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002aeee8e9 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948936.119631:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948936.119638:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948936.119643:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948936.119646:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948936.119649:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948936.119655:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006f266164 x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:597/0 lens 224/224 e 0 to 0 dl 1784948947 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948936.119666:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006f266164 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289662976:12345-192.168.202.137@tcp:400: Request processed in 384us (1320us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948936.119677:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 714 00000100:00000040:0.0:1784948936.119681:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948936.119685:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948936.119687:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948936.119693:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000005ae08312. 00000020:00000010:0.0:1784948936.119699:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948936.119704:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000020:00000040:0.0:1784948936.119711:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948936.119715:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948936.119853:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948936.119860:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002aeee8e9. 00000400:00000200:3.0:1784948936.119866:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948936.119872:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948936.119878:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:3.0:1784948936.119881:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000100:00000001:3.0:1784948936.119885:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948936.119888:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948936.119896:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948936.119952:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948936.119958:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948936.119963:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948936.119970:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948936.119988:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438600 00000400:00000200:3.0:1784948936.119998:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8885 [1] + 224 00000800:00000001:3.0:1784948936.120007:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948936.120015:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948936.120018:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948936.120022:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948936.120028:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948936.120033:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948936.120046:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948936.120069:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948936.120078:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948936.120081:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948936.120114:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948936.120120:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948936.120127:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948936.120130:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948936.120133:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f237c65a. 00000400:00000200:0.0:1784948936.120139:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:0.0:1784948936.120141:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784948936.120144:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948936.120151:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948936.120162:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.120171:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948936.120173:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948936.120176:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948936.120180:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948936.120182:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948936.120184:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948936.120187:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948936.120193:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=103 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948936.120197:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=103 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948936.120201:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948936.120202:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948936.120206:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948936.120210:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948936.120219:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.120222:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948936.120224:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948936.120226:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948936.120229:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948936.120233:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.120241:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948936.120244:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948936.120247:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948936.120250:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948936.120258:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289662976:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948936.120263:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948936.120266:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948936.120268:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289662976/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948936.120276:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948936.120279:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948936.120281:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 02000000:00000001:0.0:1784948936.120285:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948936.120286:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948936.120288:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948936.120291:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948936.120293:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:0.0:1784948936.120298:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948936.120300:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948936.120302:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000100:00000001:0.0:1784948936.120318:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948936.120319:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948936.120322:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948936.120326:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948936.120328:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948936.120331:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948936.120333:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948936.160021:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305875499, at 4305875500 00000100:00000010:1.0:1784948936.160041:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000040:3.0:1784948936.160046:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305875499, at 4305875500 00000100:00000010:3.0:1784948936.160064:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:2.0:1784948936.160092:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305875499, at 4305875500 00000100:00000040:0.0:1784948936.160100:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305875499, at 4305875500 00000100:00000010:2.0:1784948936.160107:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:0.0:1784948936.160117:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784948936.948066:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948936.948077:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948936.948088:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948936.948090:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948936.948221:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948936.948228:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948936.948233:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948937.140104:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948937.140120:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948937.140123:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948937.140129:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948937.140133:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948937.140137:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948937.140139:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948937.972002:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948937.972009:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948937.972016:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948937.972017:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948937.972052:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948937.972057:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948937.972060:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948938.164118:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948938.164133:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948938.164135:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948938.164143:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948938.164146:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948938.164150:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948938.164153:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948938.996066:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948938.996079:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948938.996089:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948938.996091:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948938.996250:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948938.996258:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948938.996263:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784948939.188075:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948939.188094:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948939.188098:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948939.188106:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948939.188110:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948939.188117:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948939.188120:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948940.020031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948940.020042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948940.020050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948940.020055:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948940.020058:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948940.020062:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948940.020066:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948940.020068:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948940.020070:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948940.020073:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948940.020201:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948940.020210:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948940.020215:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948940.212122:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948940.212140:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948940.212143:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948940.212149:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948940.212154:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948940.212159:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948940.212162:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948941.044054:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948941.044065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948941.044074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948941.044076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948941.044149:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948941.044158:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948941.044165:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948941.236127:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948941.236143:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948941.236158:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000053d87417. 00000020:00000040:1.0:1784948941.236170:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948941.236170:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948941.236188:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948941.236193:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784948941.236195:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948941.236200:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948941.236203:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.236207:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948941.236215:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948941.236221:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948941.236222:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000061dbde97. 00000100:00000001:2.0:1784948941.236226:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.236252:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948941.236273:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948941.236295:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948941.236299:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948941.236309:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053d87417] to pc [ptlrpcd_00_03+3] req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.236363:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948941.236367:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948941.236373:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948941.236378:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948941.236384:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948941.236391:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ddd830de. 00000020:00000040:1.0:1784948941.236394:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948941.236398:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784948941.236399:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948941.236401:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948941.236409:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.236413:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1784948941.236420:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e79c6895. 00000100:00000001:0.0:1784948941.236423:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948941.236423:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948941.236427:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948941.236437:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948941.236484:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784948941.236490:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.236525:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948941.236533:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.236548:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784948941.236550:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ddd830de] to pc [ptlrpcd_00_00+0] req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948941.236559:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948941.236572:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948941.236578:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948941.236585:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.236593:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948941.236594:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000053d87417 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289663104:0@lo:400: 00000100:00000040:1.0:1784948941.236605:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11218) 00000100:00000001:2.0:1784948941.236609:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948941.236625:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784948941.236628:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784948941.236630:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948941.236632:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948941.236634:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 02000000:00000001:2.0:1784948941.236637:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948941.236638:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.236641:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948941.236658:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:2.0:1784948941.236660:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 00000100:00000001:1.0:1784948941.236671:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948941.236673:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.236712:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948941.236716:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948941.236721:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:2.0:1784948941.236722:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a9a70155. 00000100:00000040:1.0:1784948941.236733:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948941.236741:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000001:1.0:1784948941.236755:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948941.236759:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948941.236761:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289663104, portal 25 00000100:00000001:2.0:1784948941.236770:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1784948941.236771:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948941.236786:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549159808 : -115544160391808 : ffff96e9c75d2d80) 02000000:00000001:1.0:1784948941.236788:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948941.236806:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948941.236811:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948941.236833:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289663232:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948941.236836:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948941.236852:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784948941.236860:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:2.0:1784948941.236861:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000100:00000200:2.0:1784948941.236873:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289663104, offset 0 02000000:00000001:1.0:1784948941.236877:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948941.236884:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948941.236885:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784948941.236898:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948941.236908:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 02000000:00000001:1.0:1784948941.236917:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948941.236998:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:1.0:1784948941.237013:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009a55be35. 00000400:00000200:2.0:1784948941.237028:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438680 00000400:00000010:1.0:1784948941.237030:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000200:2.0:1784948941.237061:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7e7d [2] + 0 00000400:00000010:2.0:1784948941.237066:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000037f9e020. 00000400:00000200:2.0:1784948941.237079:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bfafda61 00000100:00000200:1.0:1784948941.237089:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289663232, portal 10 00000400:00000200:2.0:1784948941.237092:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948941.237095:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948941.237100:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549156224 : -115544160395392 : ffff96e9c75d1f80) 00000400:00000200:2.0:1784948941.237107:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948941.237117:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bfafda61 00000100:00000040:1.0:1784948941.237122:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948941.237124:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bfafda61. 00000100:00000001:2.0:1784948941.237145:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948941.237149:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:2.0:1784948941.237150:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784948941.237158:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000074ed294d x1871651289663104 msgsize 224 00000100:00000040:1.0:1784948941.237159:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948941.237173:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000100:00000200:1.0:1784948941.237192:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289663232, offset 0 00000100:00100000:2.0:1784948941.237201:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948941.237209:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784948941.237218:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948941.237251:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948941.237252:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948941.237290:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784948941.237297:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948941.237304:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948941.237309:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784948941.237311:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948941.237320:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289663104 00000400:00000200:1.0:1784948941.237320:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3421:3421:240:4294967295] 192.168.202.137@tcp LPNI seq info [2207:2207:8:4294967295] 02000000:00000001:0.0:1784948941.237329:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948941.237361:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948941.237364:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948941.237373:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948941.237377:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:0.0:1784948941.237380:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289663104 00000020:00000001:0.0:1784948941.237386:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:1.0:1784948941.237390:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000040:0.0:1784948941.237391:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:2.0:1784948941.237397:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000020:00000001:0.0:1784948941.237398:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000010:1.0:1784948941.237399:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000020:00000040:0.0:1784948941.237406:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000010:2.0:1784948941.237412:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000020:00000001:0.0:1784948941.237414:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:1.0:1784948941.237424:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:0.0:1784948941.237425:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948941.237429:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948941.237457:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948941.237464:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000200:2.0:1784948941.237467:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784948941.237471:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784948941.237472:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948941.237494:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000800:00000200:1.0:1784948941.237496:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000020:00000010:0.0:1784948941.237506:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001879354a. 00000100:00000001:1.0:1784948941.237519:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948941.237521:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007f8dfdc6. 00000100:00000001:2.0:1784948941.237528:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948941.237530:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2106 usecs (3 jiffies) 00000020:00000010:0.0:1784948941.237531:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:2.0:1784948941.237539:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.237542:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305880580 00000100:00000040:0.0:1784948941.237549:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000010:1.0:1784948941.237558:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:0.0:1784948941.237563:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948941.237570:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948941.237573:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948941.237580:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.237591:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305880580 00000100:00000001:0.0:1784948941.237597:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.237602:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948941.237603:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784948941.237614:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948941.237618:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948941.237619:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948941.237635:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305880580 00000100:00000001:2.0:1784948941.237638:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948941.237640:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000010:1.0:1784948941.237641:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:2.0:1784948941.237648:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305880619 00000100:00000001:0.0:1784948941.237649:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948941.237651:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00100000:0.0:1784948941.237663:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 739 00000100:00000040:0.0:1784948941.237670:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948941.237674:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305880580 00000100:00000001:0.0:1784948941.237676:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741507664 : -115539968043952 : ffff96eac13f5450) 00000100:00000001:2.0:1784948941.237676:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305880580 00000100:00000010:1.0:1784948941.237710:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000040:0.0:1784948941.237713:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000074ed294d x1871651289663104/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:602/0 lens 224/0 e 0 to 0 dl 1784948952 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.237719:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948941.237719:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305880619 00000100:00000001:0.0:1784948941.237743:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948941.237749:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:1.0:1784948941.237750:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948941.237751:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305880580 00000100:00000001:0.0:1784948941.237754:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948941.237768:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.237770:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305880619 00000020:00000010:0.0:1784948941.237771:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:1.0:1784948941.237778:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.237790:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.237794:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305880580 00000100:00100000:0.0:1784948941.237801:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000074ed294d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289663104:12345-0@lo:400: 00000100:00000001:2.0:1784948941.237806:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305880619 00000100:00000200:0.0:1784948941.237808:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289663104 00000800:00000200:3.0:1784948941.237811:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1784948941.237815:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784948941.237817:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305880580 00000800:00000001:1.0:1784948941.237821:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784948941.237823:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000010:3.0:1784948941.237824:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000100:00000001:2.0:1784948941.237825:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784948941.237831:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.237838:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948941.237842:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:1.0:1784948941.237842:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.237845:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948941.237846:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784948941.237849:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784948941.237871:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948941.237883:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000001:1.0:1784948941.237884:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948941.237888:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784948941.237889:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948941.237893:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948941.237899:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.237899:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:0.0:1784948941.237902:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948941.237913:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784948941.237915:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000100:00000001:2.0:1784948941.237922:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948941.237942:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948941.237978:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:3.0:1784948941.237986:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000020:00000001:0.0:1784948941.237987:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.237990:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948941.238004:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:1.0:1784948941.238004:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438700 00000100:00000001:2.0:1784948941.238005:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948941.238008:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948941.238016:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948941.238028:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948941.238046:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001f4abcc4. 00000100:00000200:3.0:1784948941.238049:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948941.238050:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 161200 02000000:00000001:0.0:1784948941.238054:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948941.238064:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948941.238073:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784948941.238075:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948941.238085:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289663104 00000800:00000001:1.0:1784948941.238088:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784948941.238096:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:1.0:1784948941.238101:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948941.238104:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000200:0.0:1784948941.238107:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000074ed294d x1871651289663104/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:602/0 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948941.238115:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:3.0:1784948941.238121:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.238131:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948941.238138:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00010000:00000001:0.0:1784948941.238145:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000010:1.0:1784948941.238152:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000002cc70ea. 00010000:00000001:0.0:1784948941.238154:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948941.238164:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000002cc70ea x1871651289663232 msgsize 224 00000100:00001000:0.0:1784948941.238167:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=113 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948941.238170:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948941.238182:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1784948941.238184:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948941.238187:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948941.238195:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948941.238201:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948941.238206:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948941.238212:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.238212:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948941.238252:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948941.238257:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:1.0:1784948941.238258:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:0.0:1784948941.238261:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000800:00000001:1.0:1784948941.238268:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784948941.238269:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289663104, offset 224 00000400:00000200:0.0:1784948941.238290:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784948941.238319:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784948941.238324:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948941.238330:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289663232 02000000:00000001:1.0:1784948941.238334:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:0.0:1784948941.238340:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438680 00000100:00000001:1.0:1784948941.238340:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948941.238364:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948941.238371:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948941.238377:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8899 [1] + 224 00000100:00100000:1.0:1784948941.238378:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289663232 00000020:00000001:1.0:1784948941.238387:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948941.238390:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948941.238394:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:0.0:1784948941.238395:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1784948941.238398:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:0.0:1784948941.238408:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:1.0:1784948941.238409:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948941.238420:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948941.238423:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784948941.238426:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948941.238433:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000200:0.0:1784948941.238434:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.238469:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948941.238498:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000020:00000010:1.0:1784948941.238505:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f047fe3. 00000020:00000010:1.0:1784948941.238508:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000041b4b183. 00000020:00000010:1.0:1784948941.238513:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:1.0:1784948941.238521:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948941.238527:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948941.238530:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:0.0:1784948941.238533:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.238534:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948941.238545:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948941.238551:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948941.238553:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948941.238560:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948941.238566:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948941.238572:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:0.0:1784948941.238577:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784948941.238582:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948941.238584:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948941.238586:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948941.238588:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948941.238596:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948941.238597:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 715 00000100:00000040:1.0:1784948941.238605:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000040:0.0:1784948941.238606:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784948941.238612:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549163392 : -115544160388224 : ffff96e9c75d3b80) 00010000:00000001:0.0:1784948941.238613:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948941.238620:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948941.238623:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948941.238625:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000002cc70ea x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:602/0 lens 224/0 e 0 to 0 dl 1784948952 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948941.238629:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948941.238636:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000074ed294d x1871651289663104/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:602/0 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948941.238637:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948941.238652:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948941.238661:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948941.238670:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a9a70155. 00000100:00000001:1.0:1784948941.238673:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948941.238683:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000400:00000200:2.0:1784948941.238690:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000100:00000001:1.0:1784948941.238692:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784948941.238693:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000074ed294d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289663104:12345-0@lo:400: Request processed in 944us (1528us total) trans 0 rc 0/0 00000020:00000010:1.0:1784948941.238697:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000400:00000010:2.0:1784948941.238718:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:2.0:1784948941.238727:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784948941.238729:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 739 00000100:00100000:1.0:1784948941.238730:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000002cc70ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289663232:12345-192.168.202.137@tcp:400: 00000100:00000040:0.0:1784948941.238740:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:2.0:1784948941.238742:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948941.238748:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289663232 00000100:00000001:0.0:1784948941.238754:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:1.0:1784948941.238761:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948941.238763:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:1.0:1784948941.238771:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000010:0.0:1784948941.238775:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007f8dfdc6. 00000020:00000001:1.0:1784948941.238779:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948941.238788:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000001:1.0:1784948941.238797:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000010:0.0:1784948941.238801:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001879354a. 00000100:00000200:2.0:1784948941.238803:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948941.238809:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000040:0.0:1784948941.238813:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784948941.238846:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1784948941.238848:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 02000000:00000001:0.0:1784948941.238850:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948941.238852:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784948941.238859:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000001f4abcc4. 00000020:00000001:1.0:1784948941.238861:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.238861:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948941.238867:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:2.0:1784948941.238869:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948941.238876:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948941.238881:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784948941.238883:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:0.0:1784948941.238885:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948941.238896:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948941.238897:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:0.0:1784948941.238903:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f237c65a. 02000000:00000001:2.0:1784948941.238904:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948941.238906:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948941.238922:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948941.238928:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948941.238964:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000010:0.0:1784948941.238967:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b1caac62. 00000100:00001000:2.0:1784948941.238974:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=113 v=5 (5 5 5 5) 02000000:00000010:1.0:1784948941.238975:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002afa749d. 02000000:00000001:1.0:1784948941.238993:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948941.238995:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=113 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948941.239005:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.239007:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:1.0:1784948941.239014:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.239015:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948941.239027:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289663232 00010000:00000001:2.0:1784948941.239027:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784948941.239040:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948941.239041:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784948941.239054:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:1.0:1784948941.239059:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000002cc70ea x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:602/0 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948941.239065:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948941.239148:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784948941.239149:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1784948941.239161:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.239162:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948941.239176:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784948941.239178:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=108 v=5 (1 1 1 1) 00000100:00000040:2.0:1784948941.239192:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.239194:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948941.239206:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948941.239219:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948941.239257:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948941.239261:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948941.239262:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000053d87417 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289663104:0@lo:400: 00000100:00000001:1.0:1784948941.239267:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948941.239279:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948941.239282:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948941.239284:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000001:2.0:1784948941.239288:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1784948941.239292:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289663232, offset 224 00000100:00000040:2.0:1784948941.239297:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000053d87417 x1871651289663104/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948941.239315:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948941.239330:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1784948941.239338:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:2.0:1784948941.239338:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948941.239374:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 00000400:00000200:1.0:1784948941.239387:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3422:3422:240:4294967295] 192.168.202.137@tcp LPNI seq info [2208:2208:8:4294967295] 02000000:00000001:2.0:1784948941.239389:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948941.239401:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948941.239410:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:1.0:1784948941.239413:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:2.0:1784948941.239426:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000800:00000200:1.0:1784948941.239434:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000010:2.0:1784948941.239435:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000061dbde97. 00000800:00000010:1.0:1784948941.239476:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000031deb17e. 02000000:00000001:2.0:1784948941.239495:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948941.239502:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948941.239511:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000053d87417. 00000800:00000200:1.0:1784948941.239512:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948941.239532:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:2.0:1784948941.239532:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948941.239541:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948941.239544:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000031deb17e type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948941.239546:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.239566:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948941.239569:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948941.239573:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.239574:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948941.239581:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948941.239583:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948941.239586:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948941.239591:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948941.239596:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948941.239601:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000002cc70ea x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:602/0 lens 224/224 e 0 to 0 dl 1784948952 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948941.239615:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000002cc70ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289663232:12345-192.168.202.137@tcp:400: Request processed in 938us (1447us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948941.239625:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 715 00000100:00000040:1.0:1784948941.239628:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948941.239630:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948941.239633:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948941.239636:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000041b4b183. 00000020:00000010:1.0:1784948941.239640:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948941.239644:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f047fe3. 00000020:00000040:1.0:1784948941.239648:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948941.239650:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948941.239760:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948941.239767:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000031deb17e. 00000800:00000001:1.0:1784948941.239772:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948941.239775:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948941.239782:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948941.239788:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948941.239790:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948941.239793:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948941.239800:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000400:00000010:3.0:1784948941.239808:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000400:00000200:1.0:1784948941.239809:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784948941.239818:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948941.239823:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948941.239829:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438700 00000400:00000200:1.0:1784948941.239840:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x88a1 [1] + 224 00000800:00000001:1.0:1784948941.239866:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948941.239885:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948941.239890:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948941.239896:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948941.239902:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948941.239906:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948941.239945:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948941.239969:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948941.239976:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948941.239980:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.240068:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948941.240074:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948941.240081:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948941.240083:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948941.240086:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009a55be35. 00000400:00000200:2.0:1784948941.240093:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:2.0:1784948941.240095:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000100:00000001:2.0:1784948941.240099:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948941.240108:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948941.240120:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948941.240130:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948941.240132:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.240135:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948941.240137:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948941.240139:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948941.240141:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948941.240143:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948941.240151:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=108 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948941.240155:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=108 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948941.240158:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948941.240159:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948941.240161:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948941.240166:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948941.240173:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.240176:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948941.240178:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948941.240180:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948941.240181:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948941.240184:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948941.240191:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948941.240194:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948941.240196:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948941.240199:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948941.240206:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289663232:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948941.240210:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.240211:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948941.240213:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ddd830de x1871651289663232/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948941.240219:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948941.240223:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948941.240226:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 02000000:00000001:2.0:1784948941.240249:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948941.240251:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948941.240252:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948941.240254:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948941.240256:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e79c6895. 02000000:00000001:2.0:1784948941.240259:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948941.240260:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948941.240261:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ddd830de. 00000100:00000001:2.0:1784948941.240264:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948941.240265:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.240266:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948941.240271:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948941.240272:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948941.240273:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948941.240276:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948941.280073:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305880619, at 4305880620 00000100:00000010:3.0:1784948941.280088:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:1.0:1784948941.280100:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305880619, at 4305880620 00000100:00000010:1.0:1784948941.280116:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:0.0:1784948941.280140:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305880619, at 4305880620 00000100:00000040:2.0:1784948941.280141:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305880619, at 4305880620 00000100:00000010:0.0:1784948941.280171:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:2.0:1784948941.280177:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00010000:00000001:2.0:1784948942.067996:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948942.068005:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948942.068012:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948942.068014:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948942.068065:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948942.068071:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948942.068075:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948942.260005:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948942.260021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948942.260023:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948942.260029:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948942.260032:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948942.260036:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948942.260038:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948943.092059:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948943.092071:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948943.092080:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948943.092082:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948943.092217:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948943.092226:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948943.092232:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948943.284070:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948943.284092:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948943.284095:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948943.284102:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948943.284106:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948943.284113:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948943.284117:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948944.116050:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948944.116064:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948944.116075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948944.116077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948944.116174:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948944.116184:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948944.116189:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948944.308078:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948944.308096:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948944.308099:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948944.308109:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948944.308115:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948944.308122:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948944.308126:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948945.140059:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948945.140081:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948945.140091:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948945.140093:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948945.140179:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948945.140188:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948945.140193:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948945.332141:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948945.332158:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948945.332162:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948945.332170:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948945.332174:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948945.332180:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948945.332184:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948946.164049:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948946.164058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948946.164067:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948946.164069:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948946.164169:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784948946.164176:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784948946.164180:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948946.356052:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948946.356072:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948946.356078:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948946.356087:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948946.356090:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948946.356097:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948946.356099:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948946.356105:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948946.356111:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948946.356113:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000020:00000040:0.0:1784948946.356126:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948946.356133:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948946.356137:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948946.356144:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948946.356154:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000006487a147. 00000100:00000001:0.0:1784948946.356163:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948946.356176:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948946.356191:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948946.356195:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948946.356201:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c39699a8] to pc [ptlrpcd_00_01+1] req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948946.356253:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948946.356257:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:1.0:1784948946.356263:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784948946.356264:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948946.356273:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948946.356274:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948946.356280:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948946.356283:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948946.356289:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 00000100:00000040:1.0:1784948946.356293:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784948946.356297:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948946.356308:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948946.356314:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948946.356320:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948946.356324:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1784948946.356330:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000086e4d21. 00000100:00000001:1.0:1784948946.356330:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.356342:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948946.356345:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948946.356346:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948946.356355:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948946.356356:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948946.356357:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948946.356373:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.356381:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948946.356386:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948946.356388:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289663360:0@lo:400: 00000100:00000040:0.0:1784948946.356423:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59e6b47] to pc [ptlrpcd_00_02+2] req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948946.356428:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948946.356441:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784948946.356449:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948946.356459:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948946.356468:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000009f047fe3. 02000000:00000001:1.0:1784948946.356474:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.356478:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948946.356483:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dc4daa2d. 00000100:00000040:0.0:1784948946.356489:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11223) 00000400:00000010:1.0:1784948946.356494:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000001:3.0:1784948946.356506:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784948946.356507:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289663360, portal 25 00000100:00000001:3.0:1784948946.356515:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948946.356516:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948946.356522:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558302976 : -115544151248640 : ffff96e9c7e8b100) 00000100:00000001:3.0:1784948946.356525:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948946.356530:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948946.356538:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948946.356542:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948946.356543:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948946.356592:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948946.356594:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948946.356599:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948946.356601:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784948946.356606:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:1.0:1784948946.356609:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 02000000:00000001:3.0:1784948946.356612:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1784948946.356620:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289663360, offset 0 02000000:00000001:3.0:1784948946.356620:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948946.356635:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289663488:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784948946.356644:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948946.356646:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948946.356653:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948946.356657:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948946.356661:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948946.356668:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 00000400:00000200:1.0:1784948946.356669:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:3.0:1784948946.356678:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948946.356687:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000af4b2538. 00000400:00000200:1.0:1784948946.356693:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438780 00000400:00000010:3.0:1784948946.356695:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000400:00000200:1.0:1784948946.356708:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7ea1 [2] + 0 00000400:00000010:1.0:1784948946.356714:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000400:00000200:1.0:1784948946.356720:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000dd7f87d6 00000100:00000200:3.0:1784948946.356727:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289663488, portal 10 00000100:00000001:3.0:1784948946.356732:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948946.356734:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558303872 : -115544151247744 : ffff96e9c7e8b480) 00000400:00000200:1.0:1784948946.356744:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784948946.356745:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948946.356761:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948946.356773:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784948946.356775:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd7f87d6 00000400:00000010:1.0:1784948946.356782:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000dd7f87d6. 00000100:00000040:3.0:1784948946.356783:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948946.356793:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000010:3.0:1784948946.356795:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000100:00000200:1.0:1784948946.356800:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:3.0:1784948946.356803:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289663488, offset 0 00000100:00000040:1.0:1784948946.356808:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000001e0f2dc3 x1871651289663360 msgsize 224 00000400:00000200:3.0:1784948946.356817:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784948946.356830:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784948946.356834:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948946.356841:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948946.356849:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784948946.356851:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948946.356857:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948946.356865:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948946.356870:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948946.356876:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3423:3423:240:4294967295] 192.168.202.137@tcp LPNI seq info [2209:2209:8:4294967295] 00000400:00000200:3.0:1784948946.356894:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948946.356901:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948946.356940:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784948946.356945:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:1.0:1784948946.356955:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000010:3.0:1784948946.356955:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000001:2.0:1784948946.356963:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:1.0:1784948946.356966:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948946.356975:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289663360 00000800:00000200:3.0:1784948946.356976:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:2.0:1784948946.356984:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948946.356992:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:3.0:1784948946.357001:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784948946.357002:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948946.357005:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784948946.357012:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 02000000:00000001:2.0:1784948946.357016:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948946.357020:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948946.357024:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289663360 00000020:00000001:2.0:1784948946.357034:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948946.357035:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948946.357040:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948946.357041:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:3.0:1784948946.357045:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2108 usecs (3 jiffies) 00000100:00000001:1.0:1784948946.357051:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1784948946.357053:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948946.357054:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305885700 00000020:00000040:2.0:1784948946.357063:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948946.357065:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948946.357065:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000020:00000001:2.0:1784948946.357075:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948946.357079:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305885739 00000020:00000001:2.0:1784948946.357082:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948946.357084:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784948946.357096:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948946.357101:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305885700 00000020:00000010:2.0:1784948946.357105:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:1.0:1784948946.357108:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305885700 00000100:00000010:3.0:1784948946.357111:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002b896c5a. 00000100:00000001:2.0:1784948946.357114:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948946.357124:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:3.0:1784948946.357125:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948946.357126:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305885739 00000020:00000010:2.0:1784948946.357138:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006cd4526f. 00000100:00000001:3.0:1784948946.357142:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305885700 00000100:00000001:1.0:1784948946.357147:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305885700 00000020:00000010:2.0:1784948946.357149:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f12616d1. 00000100:00000010:3.0:1784948946.357151:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:1.0:1784948946.357159:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305885739 00000100:00000001:3.0:1784948946.357162:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000010:2.0:1784948946.357163:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784948946.357179:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305885700 00000100:00000001:1.0:1784948946.357185:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305885700 00000100:00000010:3.0:1784948946.357187:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000176cd51f. 00000100:00000040:2.0:1784948946.357188:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948946.357195:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305885739 00000100:00000001:2.0:1784948946.357200:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948946.357202:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948946.357206:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948946.357212:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948946.357214:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948946.357220:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305885700 00000100:00000001:3.0:1784948946.357221:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948946.357226:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948946.357233:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948946.357234:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948946.357244:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948946.357246:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948946.357248:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948946.357258:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948946.357258:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948946.357264:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948946.357266:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948946.357275:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948946.357276:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948946.357279:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948946.357281:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948946.357283:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948946.357285:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948946.357291:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948946.357293:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948946.357299:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1784948946.357302:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 740 00000100:00000001:1.0:1784948946.357306:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948946.357306:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948946.357310:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948946.357319:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948946.357321:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741512784 : -115539968038832 : ffff96eac13f6850) 00000100:00000040:2.0:1784948946.357333:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001e0f2dc3 x1871651289663360/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:607/0 lens 224/0 e 0 to 0 dl 1784948957 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948946.357347:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948946.357352:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:2.0:1784948946.357356:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948946.357360:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00100000:2.0:1784948946.357369:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001e0f2dc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289663360:12345-0@lo:400: 00000100:00000200:2.0:1784948946.357375:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289663360 00000020:00000001:2.0:1784948946.357379:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948946.357384:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948946.357388:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948946.357391:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948946.357394:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948946.357428:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948946.357431:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948946.357436:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948946.357439:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948946.357444:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948946.357447:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948946.357452:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948946.357456:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948946.357462:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002b206cc5. 02000000:00000001:2.0:1784948946.357465:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948946.357468:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948946.357472:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948946.357475:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289663360 00010000:00000001:2.0:1784948946.357480:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948946.357487:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001e0f2dc3 x1871651289663360/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:607/0 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948946.357498:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948946.357501:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948946.357506:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=118 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948946.357513:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948946.357518:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948946.357522:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784948946.357526:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948946.357529:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948946.357534:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948946.357535:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948946.357544:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000010:0.0:1784948946.357545:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000800:00000001:3.0:1784948946.357548:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784948946.357554:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:0.0:1784948946.357556:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784948946.357565:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289663360, offset 224 00000800:00000001:3.0:1784948946.357568:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948946.357573:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948946.357579:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948946.357586:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:3.0:1784948946.357590:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948946.357594:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000010:0.0:1784948946.357602:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000400:00000200:2.0:1784948946.357611:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948946.357613:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948946.357615:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948946.357629:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948946.357636:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438780 00000400:00000200:3.0:1784948946.357641:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438800 00000400:00000200:2.0:1784948946.357656:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x88b5 [1] + 224 00000400:00000200:2.0:1784948946.357674:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948946.357676:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784948946.357681:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 161424 00000100:00000040:0.0:1784948946.357693:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948946.357693:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:3.0:1784948946.357696:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948946.357708:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:3.0:1784948946.357710:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948946.357718:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948946.357721:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948946.357724:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.357737:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784948946.357739:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:2.0:1784948946.357759:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948946.357763:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948946.357769:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:3.0:1784948946.357779:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000002578e9dc. 00000100:00000040:3.0:1784948946.357785:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000002578e9dc x1871651289663488 msgsize 224 00000100:00100000:3.0:1784948946.357792:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784948946.357798:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784948946.357809:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000100:00000001:3.0:1784948946.357811:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:2.0:1784948946.357813:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:2.0:1784948946.357821:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948946.357824:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948946.357831:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948946.357835:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948946.357841:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948946.357844:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784948946.357851:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948946.357854:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948946.357858:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784948946.357861:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948946.357862:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:2.0:1784948946.357866:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001e0f2dc3 x1871651289663360/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:607/0 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948946.357868:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1784948946.357870:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289663488 00000400:00000001:1.0:1784948946.357879:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1784948946.357880:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948946.357889:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000001:1.0:1784948946.357889:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1784948946.357897:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001e0f2dc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289663360:12345-0@lo:400: Request processed in 543us (1082us total) trans 0 rc 0/0 00000400:00000010:1.0:1784948946.357899:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dc4daa2d. 00000100:00000001:0.0:1784948946.357900:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948946.357950:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 740 00000400:00000200:1.0:1784948946.357951:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 02000000:00000001:0.0:1784948946.357958:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948946.357959:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000040:2.0:1784948946.357962:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948946.357970:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784948946.357971:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289663488 00000100:00000001:2.0:1784948946.357972:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:0.0:1784948946.357980:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784948946.357981:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:1.0:1784948946.357982:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784948946.357987:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000010:2.0:1784948946.357992:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f12616d1. 00000020:00000001:0.0:1784948946.357996:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948946.358005:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000010:2.0:1784948946.358007:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000001:0.0:1784948946.358018:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000010:2.0:1784948946.358021:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006cd4526f. 00000020:00000001:0.0:1784948946.358029:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000040:2.0:1784948946.358034:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784948946.358042:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:2.0:1784948946.358047:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:2.0:1784948946.358051:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000002b206cc5. 00000020:00000010:0.0:1784948946.358052:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000200:1.0:1784948946.358055:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948946.358060:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948946.358062:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948946.358066:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948946.358072:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000200:2.0:1784948946.358078:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000010:0.0:1784948946.358085:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000400:00000010:2.0:1784948946.358092:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a9a70155. 00000020:00000010:0.0:1784948946.358098:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007f8dfdc6. 00000400:00000010:2.0:1784948946.358105:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fba7d3a1. 00000100:00000001:1.0:1784948946.358112:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784948946.358113:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784948946.358123:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948946.358127:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948946.358130:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784948946.358137:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784948946.358139:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948946.358143:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948946.358145:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948946.358150:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948946.358151:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948946.358163:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.358166:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00001000:1.0:1784948946.358169:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=118 v=5 (5 5 5 5) 00000100:00000001:0.0:1784948946.358172:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948946.358176:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948946.358180:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=118 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948946.358187:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948946.358189:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948946.358193:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784948946.358195:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.358198:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948946.358199:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948946.358214:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948946.358223:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948946.358225:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948946.358228:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948946.358231:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948946.358235:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948946.358242:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 716 00000100:00000040:1.0:1784948946.358245:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948946.358251:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948946.358260:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554952576 : -115544154599040 : ffff96e9c7b59180) 00000100:00100000:1.0:1784948946.358274:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289663360:0@lo:400: 00000100:00000040:0.0:1784948946.358278:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000002578e9dc x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:607/0 lens 224/0 e 0 to 0 dl 1784948957 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948946.358289:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948946.358297:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948946.358305:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784948946.358306:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c39699a8 x1871651289663360/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948946.358312:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:0.0:1784948946.358321:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948946.358326:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000010:0.0:1784948946.358328:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 02000000:00000001:1.0:1784948946.358331:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948946.358335:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000009f047fe3. 00000100:00100000:0.0:1784948946.358343:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000002578e9dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289663488:12345-192.168.202.137@tcp:400: 02000000:00000001:1.0:1784948946.358346:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948946.358354:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000200:0.0:1784948946.358355:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289663488 00000020:00000040:1.0:1784948946.358358:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948946.358361:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948946.358366:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:0.0:1784948946.358368:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000010:1.0:1784948946.358371:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000006487a147. 00000020:00000001:0.0:1784948946.358373:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948946.358378:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1784948946.358380:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:1.0:1784948946.358383:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1784948946.358385:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000010:1.0:1784948946.358389:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c39699a8. 00000020:00000001:0.0:1784948946.358392:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784948946.358424:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:0.0:1784948946.358429:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948946.358429:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948946.358432:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948946.358441:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948946.358445:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784948946.358446:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948946.358451:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948946.358454:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948946.358457:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784948946.358460:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948946.358462:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.358468:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948946.358473:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948946.358478:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000088e66292. 02000000:00000001:0.0:1784948946.358483:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.358486:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948946.358489:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948946.358492:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289663488 00010000:00000001:0.0:1784948946.358495:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948946.358500:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000002578e9dc x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:607/0 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948946.358511:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948946.358513:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948946.358516:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=113 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948946.358521:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948946.358523:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948946.358526:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948946.358529:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948946.358531:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948946.358534:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948946.358537:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948946.358540:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000200:0.0:1784948946.358544:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289663488, offset 224 00000400:00000200:0.0:1784948946.358550:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948946.358560:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948946.358566:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3424:3424:240:4294967295] 192.168.202.137@tcp LPNI seq info [2210:2210:8:4294967295] 00000400:00000200:0.0:1784948946.358578:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948946.358584:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948946.358588:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d1b12746. 00000800:00000200:0.0:1784948946.358594:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948946.358601:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948946.358604:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d1b12746 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948946.358621:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948946.358625:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948946.358628:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948946.358630:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948946.358632:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948946.358637:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002578e9dc x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:607/0 lens 224/224 e 0 to 0 dl 1784948957 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948946.358649:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000002578e9dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289663488:12345-192.168.202.137@tcp:400: Request processed in 342us (861us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948946.358660:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 716 00000100:00000040:0.0:1784948946.358665:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948946.358668:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948946.358670:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948946.358673:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007f8dfdc6. 00000020:00000010:0.0:1784948946.358678:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948946.358683:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000020:00000040:0.0:1784948946.358689:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948946.358692:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948946.358773:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948946.358775:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:1.0:1784948946.358781:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d1b12746. 00000400:00000200:1.0:1784948946.358789:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948946.358790:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948946.358798:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948946.358799:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948946.358805:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948946.358808:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:1.0:1784948946.358814:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000400:00000200:0.0:1784948946.358822:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784948946.358823:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948946.358828:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784948946.358845:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438800 00000400:00000200:0.0:1784948946.358855:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x88bd [1] + 224 00000800:00000001:0.0:1784948946.358862:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948946.358873:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948946.358876:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948946.358881:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948946.358886:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948946.358890:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948946.358902:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948946.358952:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948946.358960:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948946.358963:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948946.359026:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948946.359033:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948946.359041:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948946.359045:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948946.359048:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000af4b2538. 00000400:00000200:3.0:1784948946.359060:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000400:00000010:3.0:1784948946.359064:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 00000100:00000001:3.0:1784948946.359069:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948946.359076:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948946.359090:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948946.359101:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948946.359103:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948946.359105:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948946.359108:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948946.359110:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948946.359112:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948946.359115:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948946.359121:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=113 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948946.359125:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=113 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948946.359128:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948946.359130:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948946.359132:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948946.359137:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948946.359146:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948946.359150:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948946.359152:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948946.359154:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948946.359155:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948946.359159:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948946.359168:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948946.359170:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948946.359173:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948946.359176:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948946.359185:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289663488:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948946.359190:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948946.359193:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948946.359195:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f59e6b47 x1871651289663488/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948946.359203:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948946.359205:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948946.359207:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 02000000:00000001:3.0:1784948946.359211:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948946.359213:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948946.359215:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948946.359218:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948946.359220:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000086e4d21. 02000000:00000001:3.0:1784948946.359223:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948946.359225:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948946.359226:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f59e6b47. 00000100:00000001:3.0:1784948946.359230:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948946.359231:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948946.359233:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948946.359254:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948946.359257:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948946.359259:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948946.359262:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948946.399991:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305885739, at 4305885740 00000100:00000010:0.0:1784948946.400005:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000100:00000040:3.0:1784948946.400032:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305885739, at 4305885740 00000100:00000040:1.0:1784948946.400046:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305885739, at 4305885740 00000100:00000010:3.0:1784948946.400052:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000176cd51f. 00000100:00000040:2.0:1784948946.400053:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305885739, at 4305885740 00000100:00000010:1.0:1784948946.400061:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002b896c5a. 00000100:00000010:2.0:1784948946.400072:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00010000:00000001:2.0:1784948947.188044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948947.188054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948947.188061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948947.188063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948947.188195:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948947.188201:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948947.188206:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948947.380077:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948947.380094:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948947.380097:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948947.380104:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948947.380107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948947.380112:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948947.380115:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1784948947.892075:0:344046:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 1.2.3.4:988 00000400:00000100:2.0:1784948947.892152:0:344046:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 1.2.3.4@tcp at host 1.2.3.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948947.892165:0:344046:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 1.2.3.4@tcp: down 00000400:00000200:2.0:1784948947.892174:0:344046:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1784948947.892180:0:344046:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:2.0:1784948947.892184:0:344046:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008746b1e6. 00000400:00000200:2.0:1784948947.892192:0:344046:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948947.892198:0:344046:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948947.892204:0:344046:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11220 health :900 00000400:00000200:2.0:1784948947.892211:0:344046:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948947.892216:0:344046:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:2.0:1784948947.892221:0:344046:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:0.0:1784948947.892292:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948947.892304:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:0.0:1784948947.892313:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948947.892316:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948947.892320:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:0.0:1784948947.892323:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000400:00000200:0.0:1784948947.892328:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948947.892333:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:0.0:1784948947.892338:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000043c3e1d8. 00000400:00000200:0.0:1784948947.892345:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:0.0:1784948947.892348:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:0.0:1784948947.892352:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00010000:00000001:2.0:1784948948.212057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948948.212066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948948.212074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948948.212076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948948.212165:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948948.212172:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948948.212176:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948948.404032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948948.404050:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948948.404053:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948948.404060:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948948.404065:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948948.404070:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948948.404074:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948948.596083:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784948948.596096:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000090a03b90. 00000400:00000010:1.0:1784948948.596110:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009f047fe3. 00000400:00000010:1.0:1784948948.596117:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000010:1.0:1784948948.596126:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000033c8520. 00000400:00000200:1.0:1784948948.596132:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948948.596151:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948948.596159:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948948.596164:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3425:3425:241:4294967295] 192.168.202.137@tcp LPNI seq info [2211:2211:8:4294967295] 00000400:00000200:1.0:1784948948.596173:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784948948.596183:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784948948.596190:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948948.596196:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000800:00000200:1.0:1784948948.596204:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948948.596214:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948948.596220:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784948948.596246:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11222 health :900 00000800:00000200:3.0:1784948948.596477:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948948.596485:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:3.0:1784948948.596498:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948948.596509:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784948948.596518:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948948.596519:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784948948.596529:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948948.596539:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948948.596546:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948948.596553:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948948.596565:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948948.596580:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948948.596591:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948948.596602:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948948.596608:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3426:3426:241:4294967295] 192.168.202.137@tcp LPNI seq info [2212:2212:8:4294967295] 00000400:00000200:1.0:1784948948.596619:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948948.596624:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948948.596628:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000800:00000200:1.0:1784948948.596634:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948948.596639:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948948.596642:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:1.0:1784948948.596765:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948948.596770:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000400:00000200:1.0:1784948948.596777:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948948.596780:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948948.596787:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948948.596799:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948948.596800:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948948.596817:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948948.596823:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948948.596830:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948948.596840:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784948948.596855:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x88d1 00000800:00000001:3.0:1784948948.596862:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948948.596874:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948948.596876:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948948.596881:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784948948.596887:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:3.0:1784948948.596890:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000400:00000200:3.0:1784948948.596894:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784948948.596898:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000090a03b90. 00000400:00000010:3.0:1784948948.596902:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009f047fe3. 00000800:00000001:3.0:1784948948.596949:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948948.596955:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948949.236011:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948949.236018:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948949.236025:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948949.236027:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948949.236096:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948949.236102:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948949.236105:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948949.428087:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948949.428105:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948949.428107:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948949.428115:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948949.428118:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948949.428123:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948949.428126:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948950.260056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948950.260066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948950.260075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948950.260080:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948950.260083:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948950.260088:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948950.260091:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948950.260094:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948950.260096:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948950.260099:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.260228:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948950.260234:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948950.260239:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948950.452121:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948950.452132:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784948950.452148:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948925/real 1784948925] req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948950 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784948950.452174:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948950.452177:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948950.452180:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000de368614. 00000400:00000200:2.0:1784948950.452192:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:2.0:1784948950.452195:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000100:00000001:2.0:1784948950.452200:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948950.452204:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948950 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948950.452216:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948950 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948950.452226:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948950.452228:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.452231:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948950.452233:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948950.452237:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948950 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948950.452245:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.452255:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948950.452257:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.452260:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948950.452264:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948950.452267:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948950.452271:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948950 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948950.452279:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784948950.452283:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.452285:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948950.452286:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.452289:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784948950.452291:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784948950.452296:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784948950.452327:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:2.0:1784948950.452331:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784948950.452336:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948950 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948950.452346:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289662336:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784948950.452354:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.452358:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948950.452362:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b3c18d2b x1871651289662336/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948950 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948950.452396:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948950.452412:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948950.452419:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948950.452424:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000010:0.0:1784948950.452425:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 02000000:00000010:2.0:1784948950.452430:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000005b888794. 00000020:00000040:0.0:1784948950.452434:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 02000000:00000001:2.0:1784948950.452440:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784948950.452443:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1784948950.452446:0:337118:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:0.0:1784948950.452449:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000040:2.0:1784948950.452451:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784948950.452457:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:0.0:1784948950.452459:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948950.452461:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000867cea4b. 02000000:00000001:2.0:1784948950.452471:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948950.452473:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948950.452474:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b3c18d2b. 02000000:00000010:0.0:1784948950.452477:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a3cf4688. 00000100:00000001:2.0:1784948950.452479:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948950.452484:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.452488:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948950.452490:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.452493:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948950.452498:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.452501:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948950.452503:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948950.452505:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.452525:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948950.452529:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948950.452537:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59e6b47] to pc [ptlrpcd_00_03+3] req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948950.452575:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948950.452578:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948950.452582:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784948950.452586:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784948950.452588:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948950.452593:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784948950.452599:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11197 00000100:00000001:2.0:1784948950.452607:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784948950.452612:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784948950.452615:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1784948950.452622:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11196 00000100:00000001:2.0:1784948950.452623:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784948950.452635:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948950.452648:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948950.452652:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948950.452653:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948950.452658:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 02000000:00000001:2.0:1784948950.452663:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948950.452667:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948950.452668:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 02000000:00000001:2.0:1784948950.452670:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948950.452682:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948950.452686:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00100000:2.0:1784948950.452691:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289663616:0@lo:400: 00000100:00000001:0.0:1784948950.452694:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948950.452701:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784948950.452704:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948950.452709:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:2.0:1784948950.452711:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1784948950.452714:0:285579:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1784948950.452716:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948950.452719:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:2.0:1784948950.452723:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1784948950.452726:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:2.0:1784948950.452731:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 00000020:00000040:0.0:1784948950.452733:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:2.0:1784948950.452739:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948950.452741:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784948950.452748:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:2.0:1784948950.452750:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001712b5b2. 00000100:00000040:0.0:1784948950.452758:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000400:00000010:2.0:1784948950.452762:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000001:0.0:1784948950.452763:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948950.452773:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000200:2.0:1784948950.452775:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289663616, portal 25 00000100:00000001:0.0:1784948950.452778:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:1.0:1784948950.452782:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:2.0:1784948950.452783:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1784948950.452786:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:2.0:1784948950.452789:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558303872 : -115544151247744 : ffff96e9c7e8b480) 00000400:00000200:1.0:1784948950.452800:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00080000:0.0:1784948950.452801:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000040:2.0:1784948950.452802:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948950.452819:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000378c7327. 00000100:00080000:0.0:1784948950.452820:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:2.0:1784948950.452830:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948950.452831:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948950.452833:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 02000000:00000001:0.0:1784948950.452841:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1784948950.452842:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948950.452851:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bd6a7269. 00000400:00000010:2.0:1784948950.452852:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 02000000:00000001:0.0:1784948950.452853:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948950.452863:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.452868:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784948950.452870:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784948950.452877:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948950.452878:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289663616, offset 0 00000100:00000010:0.0:1784948950.452887:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000400:00000200:1.0:1784948950.452889:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000020:00000040:0.0:1784948950.452896:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:2.0:1784948950.452899:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948950.452941:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784948950.452946:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1784948950.452952:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:0.0:1784948950.452953:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.452963:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948950.452964:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948950.452971:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1784948950.452975:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:0.0:1784948950.452977:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.452985:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3427:3427:241:4294967295] 1.2.3.4@tcp LPNI seq info [102:102:8:4294967295] 02000000:00000010:0.0:1784948950.452988:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000b91d4ef5. 00000100:00000001:0.0:1784948950.452996:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.452997:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00100000:0.0:1784948950.453016:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784948950.453017:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:1.0:1784948950.453039:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000400:00000200:2.0:1784948950.453049:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948950.453052:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:1.0:1784948950.453053:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000100:00000001:0.0:1784948950.453059:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948950.453068:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:0.0:1784948950.453073:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c39699a8] to pc [ptlrpcd_rcv+-1] req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948950.453076:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438880 00000800:00000200:1.0:1784948950.453081:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 00000400:00000200:2.0:1784948950.453095:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7ec5 [2] + 0 00000800:00000200:1.1:1784948950.453103:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:2.0:1784948950.453112:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e8bdea01. 00000800:00000200:1.1:1784948950.453115:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 00000400:00000200:2.0:1784948950.453125:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c9198cec 00000800:00000100:1.0:1784948950.453132:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000800:00000010:1.0:1784948950.453136:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:2.0:1784948950.453142:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948950.453144:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.453145:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948950.453155:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:0.0:1784948950.453157:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1784948950.453166:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948950.453171:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c9198cec 00000100:00000001:0.0:1784948950.453172:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948950.453178:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784948950.453179:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000c9198cec. 00000100:00000010:0.0:1784948950.453180:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000100:00000001:3.0:1784948950.453187:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1784948950.453191:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948950.453192:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948950.453194:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:1.0:1784948950.453195:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000001:0.0:1784948950.453199:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1784948950.453202:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:1.0:1784948950.453213:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000040:3.0:1784948950.453213:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948950.453214:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000000c7e43f4 x1871651289663616 msgsize 224 02000000:00000001:0.0:1784948950.453216:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948950.453223:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.453231:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000010:0.0:1784948950.453237:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b84187a1. 00000100:00100000:2.0:1784948950.453244:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:0.0:1784948950.453249:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.453251:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:2.0:1784948950.453254:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784948950.453265:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000040:0.0:1784948950.453269:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948950.453278:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948950.453281:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784948950.453283:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000100:00000001:3.0:1784948950.453288:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948950.453293:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 02000000:00000001:3.0:1784948950.453298:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:2.0:1784948950.453301:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 02000000:00000001:3.0:1784948950.453306:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1784948950.453312:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784948950.453312:0:337122:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784948950.453318:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784948950.453322:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:2.0:1784948950.453328:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948950.453330:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000100:00000001:0.0:1784948950.453332:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1784948950.453335:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289663744:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784948950.453338:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948950.453340:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:3.0:1784948950.453355:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:1.0:1784948950.453356:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000040:0.0:1784948950.453360:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_00_00+0] req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948950.453384:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.453392:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:2.0:1784948950.453393:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948950.453395:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948950.453407:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948950.453416:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000378c7327. 00000100:00000001:3.0:1784948950.453417:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948950.453430:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948950.453435:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:3.0:1784948950.453445:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948950.453446:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.453447:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948950.453448:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:3.0:1784948950.453453:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.453460:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1784948950.453463:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11227) 00000100:00000001:3.0:1784948950.453464:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948950.453466:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:3.0:1784948950.453471:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948950.453477:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948950.453480:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.453486:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.453493:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948950.453495:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305889835 00000100:00000001:0.0:1784948950.453499:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948950.453504:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:2.0:1784948950.453507:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b9b6859c. 00000100:00000001:1.0:1784948950.453508:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948950.453515:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948950.453517:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.453523:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948950.453537:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948950.453541:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305889835 00000100:00000001:0.0:1784948950.453542:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948950.453549:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:0.0:1784948950.453552:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1784948950.453553:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003b6d02be. 00000100:00100000:3.0:1784948950.453556:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289663616 02000000:00000001:0.0:1784948950.453559:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948950.453565:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948950.453566:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1784948950.453567:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948950.453573:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:0.0:1784948950.453579:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289663872:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948950.453580:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.453588:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305889835 02000000:00000001:3.0:1784948950.453591:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.453592:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1784948950.453599:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 02000000:00000001:0.0:1784948950.453600:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1784948950.453602:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289663616 02000000:00000001:0.0:1784948950.453607:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.453610:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784948950.453611:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:0.0:1784948950.453613:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1784948950.453617:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784948950.453623:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:0.0:1784948950.453624:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 00000100:00000001:2.0:1784948950.453626:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305889835 00000020:00000040:3.0:1784948950.453630:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 02000000:00000001:0.0:1784948950.453633:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948950.453638:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e36e3817. 00000020:00000001:3.0:1784948950.453643:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948950.453646:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000010:0.0:1784948950.453647:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e6551ee. 00000020:00000001:3.0:1784948950.453656:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948950.453660:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948950.453661:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000100:00000001:3.0:1784948950.453670:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948950.453673:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948950.453675:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289663872, portal 10 00000100:00000001:2.0:1784948950.453681:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.453682:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000010:3.0:1784948950.453684:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000001:0.0:1784948950.453688:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000001:2.0:1784948950.453691:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948950.453699:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948950.453703:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784948950.453705:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948950.453710:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:3.0:1784948950.453713:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000100:00000001:2.0:1784948950.453721:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948950.453727:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948950.453732:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.453735:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:3.0:1784948950.453735:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000041892d66. 00000100:00000001:2.0:1784948950.453738:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948950.453748:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000010:3.0:1784948950.453749:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000001ceb335. 00000400:00000010:0.0:1784948950.453756:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000020:00000010:3.0:1784948950.453760:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000200:0.0:1784948950.453763:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289663872, offset 0 00000400:00000200:0.0:1784948950.453778:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784948950.453780:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948950.453787:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948950.453791:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784948950.453792:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784948950.453798:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948950.453803:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948950.453810:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948950.453812:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948950.453815:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948950.453822:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948950.453826:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3428:3428:241:4294967295] 192.168.202.137@tcp LPNI seq info [2213:2213:8:4294967295] 00000400:00000200:0.0:1784948950.453839:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948950.453839:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948950.453852:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948950.453858:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000595fa8f8. 00000100:00000001:3.0:1784948950.453861:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000200:0.0:1784948950.453869:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948950.453869:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948950.453882:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 741 00000800:00000200:0.0:1784948950.453889:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:3.0:1784948950.453891:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000800:00000200:0.0:1784948950.453900:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000595fa8f8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948950.453901:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741514832 : -115539968036784 : ffff96eac13f7050) 00000100:00000040:3.0:1784948950.453958:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000c7e43f4 x1871651289663616/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:611/0 lens 224/0 e 0 to 0 dl 1784948961 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948950.453961:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948950.453972:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2111 usecs (3 jiffies) 00000100:00000001:3.0:1784948950.453983:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948950.453984:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305889797 00000100:00000001:0.0:1784948950.453993:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305889835 00000020:00000010:3.0:1784948950.453993:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000001:3.0:1784948950.454003:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948950.454004:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305889797 00000020:00000010:3.0:1784948950.454012:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000100:00000001:0.0:1784948950.454014:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305889835 00000100:00000001:0.0:1784948950.454022:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305889797 00000100:00100000:3.0:1784948950.454029:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000c7e43f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289663616:12345-0@lo:400: 00000100:00000001:0.0:1784948950.454031:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305889835 00000100:00000001:0.0:1784948950.454042:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305889797 00000100:00000001:0.0:1784948950.454053:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305889835 00000100:00000200:3.0:1784948950.454055:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289663616 00000100:00000001:0.0:1784948950.454059:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784948950.454062:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948950.454065:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.454070:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784948950.454072:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.454076:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.454078:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948950.454085:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784948950.454090:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1784948950.454091:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784948950.454097:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:3.0:1784948950.454099:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784948950.454102:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948950.454105:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948950.454107:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784948950.454111:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784948950.454113:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.454117:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784948950.454119:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.454122:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.454127:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948950.454131:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948950.454134:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948950.454140:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008d04782c. 02000000:00000001:3.0:1784948950.454144:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948950.454147:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948950.454150:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948950.454154:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289663616 00010000:00000001:3.0:1784948950.454158:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948950.454165:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000c7e43f4 x1871651289663616/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:611/0 lens 224/224 e 0 to 0 dl 1784948961 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784948950.454168:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948950.454178:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000595fa8f8. 00000800:00000001:0.0:1784948950.454183:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:3.0:1784948950.454184:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:1.0:1784948950.454189:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:3.0:1784948950.454191:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948950.454200:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=122 v=5 (1 1 1 1) 00000800:00000001:0.0:1784948950.454202:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948950.454207:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948950.454215:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948950.454218:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784948950.454221:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000800:00000001:0.0:1784948950.454224:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948950.454226:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000010:1.0:1784948950.454229:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:3.0:1784948950.454235:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784948950.454241:0:337114:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784948950.454244:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:0.0:1784948950.454246:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:3.0:1784948950.454251:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948950.454255:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948950.454261:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784948950.454272:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438980 00000100:00000040:3.0:1784948950.454275:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948950.454289:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000100:00000001:1.0:1784948950.454292:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784948950.454295:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 161648 00000100:00000040:1.0:1784948950.454304:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948950.454306:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784948950.454312:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289663616, offset 224 00000800:00000001:0.0:1784948950.454318:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948950.454329:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948950.454331:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948950.454332:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948950.454340:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948950.454346:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948950.454358:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948950.454361:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948950.454390:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784948950.454398:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000457a2a3. 00000400:00000200:3.0:1784948950.454400:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438880 00000100:00000040:0.0:1784948950.454407:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000000457a2a3 x1871651289663872 msgsize 224 00000400:00000200:3.0:1784948950.454415:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x88d5 [1] + 224 00000100:00100000:0.0:1784948950.454425:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:3.0:1784948950.454432:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948950.454440:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948950.454445:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948950.454446:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:3.0:1784948950.454452:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948950.454461:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948950.454472:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948950.454473:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948950.454501:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948950.454504:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1784948950.454507:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289663872 02000000:00000001:1.0:1784948950.454512:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948950.454515:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:3.0:1784948950.454518:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000100:00000001:1.0:1784948950.454519:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948950.454526:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 02000000:00000001:1.0:1784948950.454528:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948950.454533:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00100000:1.0:1784948950.454535:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289663872 00000100:00000001:3.0:1784948950.454539:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:1.0:1784948950.454541:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948950.454546:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:3.0:1784948950.454548:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948950.454550:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948950.454559:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:3.0:1784948950.454561:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:1.0:1784948950.454567:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:3.0:1784948950.454569:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948950.454576:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948950.454577:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948950.454581:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784948950.454582:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784948950.454585:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948950.454590:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1784948950.454594:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000040:3.0:1784948950.454595:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c7e43f4 x1871651289663616/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:611/0 lens 224/224 e 0 to 0 dl 1784948961 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784948950.454602:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784948950.454604:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1784948950.454610:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:1.0:1784948950.454617:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000400:00000010:2.0:1784948950.454619:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001712b5b2. 00000020:00000010:1.0:1784948950.454630:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 00000400:00000200:2.0:1784948950.454633:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000020:00000010:1.0:1784948950.454641:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006e1d5987. 00000100:00100000:3.0:1784948950.454643:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000c7e43f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289663616:12345-0@lo:400: Request processed in 651us (1420us total) trans 0 rc 0/0 00000400:00000010:2.0:1784948950.454644:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000020:00000010:1.0:1784948950.454653:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948950.454656:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948950.454668:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948950.454671:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948950.454680:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00100000:3.0:1784948950.454682:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 741 00000100:00000001:1.0:1784948950.454686:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948950.454692:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948950.454698:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948950.454705:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:2.0:1784948950.454713:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948950.454714:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948950.454714:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948950.454720:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948950.454725:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948950.454738:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000001ceb335. 00000100:00000001:2.0:1784948950.454739:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948950.454745:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948950.454747:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948950.454749:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:2.0:1784948950.454751:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948950.454760:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:3.0:1784948950.454762:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000041892d66. 00000100:00000001:2.0:1784948950.454764:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948950.454768:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948950.454772:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948950.454774:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000040:3.0:1784948950.454777:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:2.0:1784948950.454780:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948950.454788:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00100000:1.0:1784948950.454790:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 717 00000100:00001000:2.0:1784948950.454793:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=122 v=5 (5 5 5 5) 02000000:00000010:3.0:1784948950.454795:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008d04782c. 00000100:00000040:1.0:1784948950.454798:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:3.0:1784948950.454806:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784948950.454808:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529525959168 : -115544183592448 : ffff96e9c5fb2a00) 00000100:00001000:2.0:1784948950.454811:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=122 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948950.454814:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948950.454824:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000457a2a3 x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:611/0 lens 224/0 e 0 to 0 dl 1784948961 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948950.454828:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000200:3.0:1784948950.454829:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948950.454835:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948950.454841:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000af4b2538. 00010000:00000001:2.0:1784948950.454847:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784948950.454853:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948950.454855:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000034386f9e. 00000100:00000001:1.0:1784948950.454863:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948950.454865:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948950.454872:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000040:2.0:1784948950.454878:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948950.454880:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948950.454889:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00000001:2.0:1784948950.454936:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948950.454941:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948950.454944:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948950.454946:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000457a2a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289663872:12345-192.168.202.137@tcp:400: 00000100:00000040:2.0:1784948950.454955:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948950.454958:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289663872 00000020:00000001:1.0:1784948950.454964:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948950.454971:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948950.454978:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948950.454980:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289663616:0@lo:400: 00000020:00000001:1.0:1784948950.454984:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948950.454990:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784948950.454994:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948950.454996:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948950.455002:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948950.455003:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784948950.455011:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:2.0:1784948950.455012:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f59e6b47 x1871651289663616/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948950.455015:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948950.455021:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948950.455029:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948950.455035:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948950.455039:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948950.455040:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:2.0:1784948950.455045:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948950.455048:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009265644e. 02000000:00000010:2.0:1784948950.455050:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 02000000:00000001:1.0:1784948950.455056:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948950.455061:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1784948950.455062:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948950.455068:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:1.0:1784948950.455069:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784948950.455074:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00010000:00000040:1.0:1784948950.455076:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289663872 00000020:00000001:2.0:1784948950.455087:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000001:1.0:1784948950.455088:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000010:2.0:1784948950.455096:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a3cf4688. 00010000:00000200:1.0:1784948950.455100:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000457a2a3 x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:611/0 lens 224/224 e 0 to 0 dl 1784948961 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948950.455105:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948950.455111:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948950.455116:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f59e6b47. 00010000:00000001:1.0:1784948950.455121:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784948950.455126:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00010000:00000001:1.0:1784948950.455127:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948950.455132:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784948950.455134:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=117 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948950.455138:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948950.455143:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948950.455149:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784948950.455150:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948950.455156:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948950.455158:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784948950.455163:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948950.455164:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784948950.455168:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948950.455169:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948950.455180:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948950.455184:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948950.455187:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000200:1.0:1784948950.455191:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289663872, offset 224 00000400:00000200:1.0:1784948950.455201:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948950.455211:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948950.455216:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3429:3429:241:4294967295] 192.168.202.137@tcp LPNI seq info [2214:2214:8:4294967295] 00000400:00000200:1.0:1784948950.455228:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948950.455235:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948950.455239:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000800:00000200:1.0:1784948950.455244:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948950.455250:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948950.455254:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948950.455270:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948950.455274:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948950.455277:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948950.455279:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948950.455281:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948950.455285:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000457a2a3 x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:611/0 lens 224/224 e 0 to 0 dl 1784948961 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948950.455296:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000457a2a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289663872:12345-192.168.202.137@tcp:400: Request processed in 432us (888us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948950.455305:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 717 00000100:00000040:1.0:1784948950.455308:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948950.455310:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948950.455312:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948950.455315:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006e1d5987. 00000020:00000010:1.0:1784948950.455320:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948950.455323:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00000020:00000040:1.0:1784948950.455327:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948950.455330:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948950.455518:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948950.455526:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000400:00000200:3.0:1784948950.455534:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948950.455544:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948950.455546:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948950.455550:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:3.0:1784948950.455555:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000800:00000001:1.0:1784948950.455560:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948950.455561:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784948950.455568:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948950.455569:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784948950.455577:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948950.455588:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948950.455603:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438980 00000400:00000200:1.0:1784948950.455611:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x88e1 [1] + 224 00000800:00000001:1.0:1784948950.455620:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948950.455633:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948950.455636:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948950.455640:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948950.455645:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948950.455650:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948950.455661:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948950.455682:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948950.455689:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948950.455691:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948950.455792:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948950.455798:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948950.455805:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948950.455807:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948950.455810:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e6551ee. 00000400:00000200:0.0:1784948950.455818:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:0.0:1784948950.455820:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000001:0.0:1784948950.455824:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948950.455831:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948950.455845:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948950.455855:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948950.455857:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948950.455860:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948950.455862:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948950.455864:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948950.455866:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948950.455868:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948950.455874:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=117 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948950.455878:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=117 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948950.455881:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948950.455883:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948950.455885:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948950.455889:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948950.455898:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.455902:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948950.455935:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948950.455937:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948950.455939:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948950.455943:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948950.455951:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948950.455954:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948950.455956:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948950.455960:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948950.455977:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289663872:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948950.455983:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948950.455985:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948950.455987:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b3c18d2b x1871651289663872/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948966 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948950.455995:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948950.455998:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948950.456000:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784948950.456005:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948950.456007:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948950.456009:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948950.456011:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948950.456014:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b84187a1. 02000000:00000001:0.0:1784948950.456016:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948950.456018:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948950.456019:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b3c18d2b. 00000100:00000001:0.0:1784948950.456023:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948950.456024:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948950.456026:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948950.456031:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948950.456033:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948950.456036:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948950.456039:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948950.496000:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305889835, at 4305889836 00000100:00000010:3.0:1784948950.496016:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e36e3817. 00000100:00000040:1.0:1784948950.496019:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305889835, at 4305889836 00000100:00000010:1.0:1784948950.496035:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003b6d02be. 00000100:00000040:2.0:1784948950.496066:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305889835, at 4305889836 00000100:00000040:0.0:1784948950.496067:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305889835, at 4305889836 00000100:00000010:2.0:1784948950.496085:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000010:0.0:1784948950.496088:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b9b6859c. 00010000:00000001:2.0:1784948951.284057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948951.284067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948951.284075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948951.284077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948951.284205:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948951.284212:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948951.284216:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948951.476015:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948951.476025:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1784948951.476043:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948950/real 0] req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948951 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.476074:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.476078:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948951.476081:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948951.476087:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948951 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.476098:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.476110:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948951.476112:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.476116:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948951.476120:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948951.476122:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948951.476126:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948951 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.476144:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1784948951.476148:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.476150:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948951.476152:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.476156:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1784948951.476158:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1784948951.476162:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1784948951.476196:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:3.0:1784948951.476200:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1784948951.476205:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948951 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948951.476215:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c39699a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289663744:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784948951.476223:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.476227:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948951.476231:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c39699a8 x1871651289663744/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948951 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.476241:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1784948951.476245:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948951.476248:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:0.0:1784948951.476251:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1784948951.476252:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948951.476261:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000b91d4ef5. 00000100:00000001:0.0:1784948951.476263:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:3.0:1784948951.476271:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000010:0.0:1784948951.476276:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b93d109. 02000000:00000001:3.0:1784948951.476277:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948951.476282:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c39699a8. 00000020:00000040:0.0:1784948951.476286:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948951.476292:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948951.476296:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948951.476297:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.476304:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948951.476305:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948951.476309:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1784948951.476314:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.476315:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.476326:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948951.476339:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948951.476343:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b84187a1. 00000100:00000001:0.0:1784948951.476428:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948951.476438:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948951.476454:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948951.476457:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948951.476464:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005b93d109] to pc [ptlrpcd_00_01+1] req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948951.476506:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948951.476510:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948951.476515:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784948951.476519:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784948951.476522:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948951.476527:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784948951.476534:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11197 00000100:00000001:2.0:1784948951.476537:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948951.476547:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948951.476550:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784948951.476553:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1784948951.476560:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11222 00000100:00000001:3.0:1784948951.476561:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948951.476568:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.476571:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948951.476589:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:2.0:1784948951.476592:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948951.476597:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948951.476599:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00080000:0.0:1784948951.476608:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1784948951.476609:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784948951.476625:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948951.476627:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:2.0:1784948951.476637:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.476638:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948951.476645:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00100000:2.0:1784948951.476650:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289664000:0@lo:400: 00000100:00000001:0.0:1784948951.476652:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:0.0:1784948951.476661:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784948951.476663:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1784948951.476666:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784948951.476671:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1784948951.476673:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1784948951.476677:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:2.0:1784948951.476678:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948951.476688:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:2.0:1784948951.476689:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1784948951.476696:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000010:2.0:1784948951.476701:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 00000100:00000040:0.0:1784948951.476703:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000001:2.0:1784948951.476710:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.476711:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948951.476719:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000400:00000010:2.0:1784948951.476722:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001712b5b2. 00000100:00000001:0.0:1784948951.476724:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000200:1.0:1784948951.476727:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000040:0.0:1784948951.476734:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:2.0:1784948951.476738:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:1.0:1784948951.476742:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00080000:0.0:1784948951.476747:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000200:2.0:1784948951.476754:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289664000, portal 25 00000100:00080000:0.0:1784948951.476755:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000400:00000010:1.0:1784948951.476758:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000378c7327. 00000100:00000001:2.0:1784948951.476763:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948951.476770:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948951.476771:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451647872 : -115544257903744 : ffff96e9c18d4380) 00000400:00000010:1.0:1784948951.476773:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 02000000:00000001:0.0:1784948951.476782:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1784948951.476790:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948951.476791:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001fcbecf4. 02000000:00000001:0.0:1784948951.476793:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948951.476805:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948951.476811:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784948951.476814:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784948951.476823:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948951.476833:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e8f14423. 00000100:00000001:2.0:1784948951.476835:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784948951.476840:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000020:00000040:0.0:1784948951.476844:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:1.0:1784948951.476851:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1784948951.476854:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784948951.476857:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1784948951.476863:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:0.0:1784948951.476864:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948951.476865:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000400:00000200:1.0:1784948951.476874:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948951.476876:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1784948951.476881:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289664000, offset 0 02000000:00000001:0.0:1784948951.476882:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1784948951.476883:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:0.0:1784948951.476890:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948951.476898:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3430:3430:241:4294967295] 1.2.3.4@tcp LPNI seq info [103:103:8:4294967295] 02000000:00000010:0.0:1784948951.476935:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000b91d4ef5. 00000100:00000001:0.0:1784948951.476943:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948951.476951:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000400:00000200:2.0:1784948951.476959:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:0.0:1784948951.476960:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784948951.476978:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:1.0:1784948951.477003:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784948951.477007:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948951.477014:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784948951.477014:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000400:00000200:2.0:1784948951.477017:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:1.0:1784948951.477034:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:0.0:1784948951.477045:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e8f14423] to pc [ptlrpcd_rcv+-1] req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784948951.477046:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 56 secs later) 00000400:00000200:2.0:1784948951.477059:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438a00 00000800:00000200:1.1:1784948951.477067:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784948951.477080:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 56 secs later) 00000400:00000200:2.0:1784948951.477089:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7f59 [2] + 0 00000800:00000100:1.0:1784948951.477097:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784948951.477103:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784948951.477106:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000010:2.0:1784948951.477107:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000100:00000040:0.0:1784948951.477112:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1784948951.477117:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948951.477124:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000003f136305 00000100:00000001:0.0:1784948951.477127:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:1.0:1784948951.477131:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784948951.477140:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948951.477146:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948951.477149:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:1.0:1784948951.477164:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:2.0:1784948951.477167:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948951.477183:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000400:00000200:2.0:1784948951.477186:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f136305 00000400:00000200:1.0:1784948951.477192:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000400:00000010:2.0:1784948951.477197:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000003f136305. 00000100:00000001:3.0:1784948951.477205:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948951.477206:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000100:00000001:2.0:1784948951.477215:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000001:1.0:1784948951.477216:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948951.477223:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:2.0:1784948951.477226:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:1.0:1784948951.477228:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784948951.477234:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000040:2.0:1784948951.477240:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000093c39287 x1871651289664000 msgsize 224 00000100:00000001:3.0:1784948951.477241:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948951.477243:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:3.0:1784948951.477248:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1784948951.477255:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000040:3.0:1784948951.477265:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948951.477267:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000378c7327. 00000100:00100000:2.0:1784948951.477277:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:1.0:1784948951.477286:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000040:2.0:1784948951.477292:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784948951.477301:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:3.0:1784948951.477303:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948951.477315:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948951.477317:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 02000000:00000001:3.0:1784948951.477322:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948951.477327:0:337120:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:3.0:1784948951.477329:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1784948951.477333:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e19c9a62. 02000000:00000001:3.0:1784948951.477337:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948951.477342:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:2.0:1784948951.477345:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000100:00100000:3.0:1784948951.477348:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289664128:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784948951.477353:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948951.477391:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1784948951.477395:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 02000000:00000001:0.0:1784948951.477398:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.477398:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784948951.477406:0:337120:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784948951.477407:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:0.0:1784948951.477411:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000000d1fbdb0. 02000000:00000001:3.0:1784948951.477415:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.477421:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948951.477422:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948951.477423:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784948951.477434:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d8bbc1a8. 00000100:00000040:0.0:1784948951.477440:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1784948951.477445:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948951.477450:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1784948951.477458:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009ef01e88. 00000100:00000040:2.0:1784948951.477462:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948951.477473:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000100:00000001:0.0:1784948951.477474:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948951.477480:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948951.477486:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289664128, portal 4 00000100:00000001:3.0:1784948951.477495:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1784948951.477497:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e19c9a62] to pc [ptlrpcd_00_02+2] req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.477501:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529451655040 : -115544257896576 : ffff96e9c18d5f80) 00000100:00000001:1.0:1784948951.477510:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948951.477517:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948951.477519:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289664000 00000100:00000040:3.0:1784948951.477525:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948951.477528:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948951.477532:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948951.477535:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948951.477542:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.477545:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948951.477550:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.477553:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784948951.477555:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948951.477557:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11228) 00000100:00000040:3.0:1784948951.477566:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948951.477568:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305890859 00000400:00000010:3.0:1784948951.477573:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 00000100:00000200:3.0:1784948951.477580:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289664128, offset 0 00000100:00000001:0.0:1784948951.477583:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:2.0:1784948951.477585:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 00000100:00000001:0.0:1784948951.477591:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784948951.477596:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948951.477598:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948951.477615:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000200:3.0:1784948951.477616:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:0.0:1784948951.477622:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948951.477634:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784948951.477635:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305890859 00000100:00100000:1.0:1784948951.477638:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289664000 00000400:00000200:3.0:1784948951.477649:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:1.0:1784948951.477650:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948951.477651:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:2.0:1784948951.477652:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f99e6ef2. 00000020:00000040:1.0:1784948951.477658:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948951.477659:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948951.477662:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784948951.477664:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1784948951.477668:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:0.0:1784948951.477672:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1784948951.477677:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784948951.477678:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305890859 00000400:00000200:3.0:1784948951.477679:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:0.0:1784948951.477680:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784948951.477690:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:2.0:1784948951.477692:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 02000000:00000001:0.0:1784948951.477693:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948951.477695:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3431:3431:241:4294967295] 192.168.202.137@tcp LPNI seq info [2215:2215:8:4294967295] 00000020:00000001:1.0:1784948951.477708:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784948951.477710:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00100000:0.0:1784948951.477712:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e19c9a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289664256:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948951.477723:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948951.477724:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305890859 00000100:00000001:0.0:1784948951.477731:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1784948951.477738:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008cd34e64. 00000400:00000200:3.0:1784948951.477739:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:0.0:1784948951.477741:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948951.477748:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948951.477753:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:0.0:1784948951.477757:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948951.477767:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1784948951.477767:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784948951.477768:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784948951.477781:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948951.477782:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948951.477784:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000020:00000010:1.0:1784948951.477785:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004204c760. 00000400:00000010:0.0:1784948951.477792:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e6551ee. 00000100:00000001:2.0:1784948951.477797:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948951.477798:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948951.477803:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000800:00000200:3.0:1784948951.477808:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784948951.477810:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948951.477812:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004204c760. 00000100:00000200:0.0:1784948951.477816:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289664256, portal 10 00000100:00000001:0.0:1784948951.477824:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948951.477827:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1784948951.477828:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 00000100:00000001:0.0:1784948951.477829:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554973440 : -115544154578176 : ffff96e9c7b5e300) 00000800:00000200:3.0:1784948951.477830:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784948951.477833:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:1.0:1784948951.477841:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006e1d5987. 00000800:00000200:3.0:1784948951.477844:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1784948951.477846:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1784948951.477851:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:0.0:1784948951.477853:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948951.477854:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948951.477863:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948951.477871:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948951.477873:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.477874:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.477876:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784948951.477883:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:3.0:1784948951.477889:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000040:0.0:1784948951.477890:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948951.477891:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948951.477900:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.477902:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305890859 00000400:00000010:0.0:1784948951.477904:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000100:00000001:2.0:1784948951.477951:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948951.477952:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948951.477959:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289664256, offset 0 00000100:00000001:3.0:1784948951.477961:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305890859 00000100:00000001:2.0:1784948951.477962:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948951.477973:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948951.477975:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948951.477976:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305890860 00000100:00000001:3.0:1784948951.477993:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305890859 00000100:00000001:1.0:1784948951.478000:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784948951.478001:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784948951.478006:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305890860 00000100:00000001:1.0:1784948951.478009:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948951.478018:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784948951.478019:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948951.478022:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305890859 00000100:00100000:1.0:1784948951.478031:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 742 00000100:00000001:2.0:1784948951.478032:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948951.478034:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305890860 00000400:00000200:0.0:1784948951.478038:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948951.478045:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305890859 00000100:00000040:1.0:1784948951.478051:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948951.478054:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948951.478056:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948951.478059:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741509712 : -115539968041904 : ffff96eac13f5c50) 00000400:00000200:0.0:1784948951.478060:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784948951.478065:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948951.478069:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.478077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948951.478078:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:1.0:1784948951.478079:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000093c39287 x1871651289664000/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:612/0 lens 224/0 e 0 to 0 dl 1784948962 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.478089:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948951.478098:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3432:3432:240:4294967295] 192.168.202.137@tcp LPNI seq info [2216:2216:7:4294967295] 00000100:00000001:3.0:1784948951.478104:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948951.478112:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1784948951.478121:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948951.478127:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948951.478130:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948951.478134:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948951.478138:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:1.0:1784948951.478139:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004204c760. 00000100:00000001:3.0:1784948951.478141:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948951.478150:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948951.478158:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004204c760. 00000800:00000200:0.0:1784948951.478165:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784948951.478172:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000093c39287 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289664000:12345-0@lo:400: 00000800:00000010:0.0:1784948951.478175:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000100:00000200:1.0:1784948951.478186:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289664000 00000800:00000200:0.0:1784948951.478194:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:1.0:1784948951.478195:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948951.478205:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948951.478211:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948951.478213:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000020:00000001:1.0:1784948951.478218:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000200:0.0:1784948951.478222:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:1.0:1784948951.478224:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948951.478234:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948951.478236:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948951.478240:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948951.478244:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948951.478247:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.478248:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948951.478254:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:1.0:1784948951.478255:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:0.0:1784948951.478257:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2112 usecs (3 jiffies) 00000100:00000001:1.0:1784948951.478262:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948951.478268:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305890821 00000800:00000010:2.0:1784948951.478268:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 02000000:00000001:1.0:1784948951.478270:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784948951.478281:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305890859 02000000:00000010:1.0:1784948951.478282:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a20b4aa7. 00000400:00000200:2.0:1784948951.478286:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948951.478291:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305890821 02000000:00000001:1.0:1784948951.478292:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.478303:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305890860 00000400:00000200:2.0:1784948951.478306:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948951.478314:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948951.478319:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948951.478321:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305890821 00000020:00000001:1.0:1784948951.478324:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948951.478330:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096db69f7 00000100:00000001:0.0:1784948951.478334:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305890860 00010000:00000040:1.0:1784948951.478334:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289664000 00000400:00000010:2.0:1784948951.478336:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000096db69f7. 00000800:00000001:3.0:1784948951.478338:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948951.478347:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948951.478348:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305890821 00000800:00000001:3.0:1784948951.478349:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948951.478352:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948951.478379:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305890860 00010000:00000200:1.0:1784948951.478382:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000093c39287 x1871651289664000/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:612/0 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948951.478386:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948951.478387:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784948951.478389:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948951.478400:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.478411:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948951.478418:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:1.0:1784948951.478421:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948951.478422:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948951.478429:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.478440:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00001000:1.0:1784948951.478441:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=123 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948951.478446:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948951.478448:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:3.0:1784948951.478453:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6438a80 00000100:00000001:1.0:1784948951.478460:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948951.478461:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948951.478461:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784948976 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948951.478469:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948951.478473:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000100:3.0:1784948951.478476:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289664128 offset 0 length 520: 4 00000100:00000001:0.0:1784948951.478477:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948951.478484:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:0.0:1784948951.478489:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948951.478497:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000006d770df4 is in inconsistent state, don't perform health checking (-2, 0) 02000000:00000001:1.0:1784948951.478498:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948951.478504:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948951.478509:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:2.0:1784948951.478516:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948951.478517:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:3.0:1784948951.478524:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948951.478525:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784948951.478527:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948951.478537:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000200:1.0:1784948951.478544:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289664000, offset 224 00000400:00000200:1.0:1784948951.478555:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948951.478570:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784948951.478584:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438a00 00000400:00000200:1.0:1784948951.478591:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x88f5 [1] + 224 00000400:00000200:1.0:1784948951.478600:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948951.478604:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948951.478621:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948951.478626:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948951.478639:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948951.478670:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000200:0.0:1784948951.478673:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1784948951.478680:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000800:00000010:0.0:1784948951.478681:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000400:00000010:1.0:1784948951.478686:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000400:00000200:0.0:1784948951.478692:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948951.478695:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948951.478696:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948951.478704:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784948951.478708:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948951.478712:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948951.478713:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948951.478718:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000800:00000001:2.0:1784948951.478725:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948951.478725:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784948951.478726:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000100:00000040:1.0:1784948951.478727:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784948951.478735:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:2.0:1784948951.478737:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784948951.478738:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784948951.478741:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784948951.478745:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784948951.478746:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948951.478753:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784948951.478757:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:0.0:1784948951.478758:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948951.478760:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001712b5b2. 00000400:00000200:2.0:1784948951.478765:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:1.0:1784948951.478772:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000093c39287 x1871651289664000/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:612/0 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948951.478774:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:3.0:1784948951.478784:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:3.0:1784948951.478795:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1784948951.478801:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438b00 00000100:00000200:3.0:1784948951.478809:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948951.478828:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948951.478830:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 161872 00000100:00000040:0.0:1784948951.478841:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948951.478848:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000093c39287 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289664000:12345-0@lo:400: Request processed in 708us (1598us total) trans 0 rc 0/0 00000800:00000001:2.0:1784948951.478854:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948951.478871:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948951.478884:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:3.0:1784948951.478892:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948951.478893:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 742 00000100:00000001:0.0:1784948951.478894:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.478905:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784948951.478947:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948951.478951:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948951.478954:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948951.478957:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948951.478961:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948951.478962:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948951.478963:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948951.478972:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:1.0:1784948951.478974:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006e1d5987. 00000100:00000001:3.0:1784948951.478976:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948951.478977:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784948951.478985:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948951.478986:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000010:1.0:1784948951.478987:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000800:00000001:0.0:1784948951.478990:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784948951.478994:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948951.479002:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00000100:00000010:2.0:1784948951.479003:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008d623923. 00000800:00000001:0.0:1784948951.479004:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:3.0:1784948951.479005:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=123 v=5 (5 5 5 5) 00000020:00000040:1.0:1784948951.479013:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000800:00000001:0.0:1784948951.479015:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948951.479016:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000008d623923 x1871651289664256 msgsize 224 00000100:00001000:3.0:1784948951.479018:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=123 v=5 (1 1 1 1) 02000000:00000001:1.0:1784948951.479026:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948951.479038:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:1.0:1784948951.479042:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a20b4aa7. 00000100:00100000:2.0:1784948951.479044:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784948951.479045:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948951.479052:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:3.0:1784948951.479057:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:1.0:1784948951.479058:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784948951.479063:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948951.479067:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948951.479070:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784948951.479071:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948951.479077:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dc4daa2d. 00000100:00000040:3.0:1784948951.479084:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948951.479087:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784948951.479088:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007790144d. 00000800:00000001:2.0:1784948951.479101:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.479104:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948951.479110:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948951.479111:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948951.479119:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948951.479120:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289664256 02000000:00000001:0.0:1784948951.479129:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:3.0:1784948951.479132:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948951.479134:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948951.479142:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948951.479151:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948951.479160:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289664256 00000100:00100000:3.0:1784948951.479161:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005b93d109 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289664000:0@lo:400: 00000020:00000001:0.0:1784948951.479169:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948951.479174:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:3.0:1784948951.479174:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.479183:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784948951.479184:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948951.479191:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:3.0:1784948951.479192:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005b93d109 x1871651289664000/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948951.479199:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948951.479208:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948951.479220:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948951.479221:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948951.479227:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:0.0:1784948951.479228:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 02000000:00000010:3.0:1784948951.479230:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 00000100:00000001:0.0:1784948951.479239:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948951.479241:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948951.479247:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:0.0:1784948951.479249:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000020:00000040:3.0:1784948951.479252:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784948951.479258:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000010:0.0:1784948951.479260:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001879354a. 02000000:00000010:3.0:1784948951.479261:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b84187a1. 00000020:00000010:0.0:1784948951.479273:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007f8dfdc6. 02000000:00000001:3.0:1784948951.479274:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948951.479279:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948951.479283:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b93d109. 00000020:00000010:0.0:1784948951.479284:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784948951.479290:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948951.479295:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.479298:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948951.479300:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784948951.479304:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948951.479307:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948951.479309:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.479313:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784948951.479315:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948951.479319:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.479321:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.479331:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948951.479336:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948951.479339:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.479376:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948951.479389:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948951.479392:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948951.479399:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 718 00000100:00000040:0.0:1784948951.479403:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948951.479406:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470027776 : -115544239523840 : ffff96e9c2a5b800) 00000100:00000040:0.0:1784948951.479413:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000008d623923 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:612/0 lens 224/0 e 0 to 0 dl 1784948962 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948951.479425:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948951.479429:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:0.0:1784948951.479433:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948951.479436:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00100000:0.0:1784948951.479441:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000008d623923 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289664256:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784948951.479446:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289664256 00000020:00000001:0.0:1784948951.479449:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948951.479452:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948951.479454:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948951.479457:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948951.479459:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784948951.479463:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948951.479466:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948951.479468:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948951.479470:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948951.479472:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948951.479474:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948951.479478:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948951.479480:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948951.479484:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000867cea4b. 02000000:00000001:0.0:1784948951.479486:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.479488:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948951.479491:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948951.479493:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289664256 00010000:00000001:0.0:1784948951.479496:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948951.479500:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000008d623923 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:612/0 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948951.479509:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948951.479511:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948951.479514:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=118 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948951.479518:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948951.479521:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948951.479524:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948951.479527:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948951.479529:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948951.479532:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948951.479535:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948951.479538:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000100:00000200:0.0:1784948951.479541:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289664256, offset 224 00000400:00000200:0.0:1784948951.479546:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948951.479555:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948951.479560:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3433:3433:241:4294967295] 192.168.202.137@tcp LPNI seq info [2217:2217:8:4294967295] 00000400:00000200:0.0:1784948951.479571:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948951.479577:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948951.479581:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000800:00000200:0.0:1784948951.479586:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948951.479592:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784948951.479596:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948951.479613:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948951.479617:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948951.479619:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948951.479621:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948951.479624:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948951.479627:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008d623923 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:612/0 lens 224/224 e 0 to 0 dl 1784948962 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948951.479639:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000008d623923 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289664256:12345-192.168.202.137@tcp:400: Request processed in 212us (621us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948951.479648:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 718 00000100:00000040:0.0:1784948951.479652:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948951.479656:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948951.479658:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948951.479661:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007f8dfdc6. 00000020:00000010:0.0:1784948951.479666:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948951.479670:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001879354a. 00000020:00000040:0.0:1784948951.479674:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948951.479677:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948951.479734:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948951.479740:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000400:00000200:1.0:1784948951.479748:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948951.479752:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948951.479759:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948951.479766:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948951.479766:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000400:00000010:1.0:1784948951.479773:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000800:00000001:0.0:1784948951.479775:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948951.479779:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784948951.479782:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948951.479783:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784948951.479799:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948951.479814:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438b00 00000400:00000200:0.0:1784948951.479821:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8909 [1] + 224 00000800:00000001:0.0:1784948951.479828:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948951.479833:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948951.479835:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948951.479840:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948951.479845:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948951.479849:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948951.479860:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948951.479883:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948951.479890:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948951.479893:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.479986:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948951.479992:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948951.479999:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948951.480001:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948951.480005:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e6551ee. 00000400:00000200:3.0:1784948951.480013:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:3.0:1784948951.480015:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000001:3.0:1784948951.480020:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948951.480027:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948951.480040:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.480051:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948951.480053:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.480055:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948951.480058:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948951.480060:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948951.480062:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948951.480065:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948951.480071:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=118 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948951.480075:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=118 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948951.480078:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948951.480079:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948951.480082:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948951.480086:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948951.480094:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.480098:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948951.480100:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948951.480102:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948951.480103:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948951.480106:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.480115:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948951.480117:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948951.480119:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948951.480123:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948951.480140:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e19c9a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289664256:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948951.480145:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.480147:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948951.480150:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e19c9a62 x1871651289664256/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948951.480157:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948951.480159:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948951.480161:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:3.0:1784948951.480165:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948951.480167:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948951.480169:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948951.480171:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948951.480173:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000000d1fbdb0. 02000000:00000001:3.0:1784948951.480176:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948951.480177:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948951.480179:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e19c9a62. 00000100:00000001:3.0:1784948951.480191:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948951.480192:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.480194:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948951.480200:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948951.480202:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948951.480204:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948951.480207:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948951.520029:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305890859, at 4305890860 00000100:00000010:0.0:1784948951.520042:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000040:3.0:1784948951.521002:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305890860, at 4305890861 00000100:00000010:3.0:1784948951.521015:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00000100:00000040:1.0:1784948951.521028:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305890860, at 4305890861 00000100:00000040:2.0:1784948951.521046:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305890860, at 4305890861 00000100:00000010:1.0:1784948951.521048:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f99e6ef2. 00000100:00000010:2.0:1784948951.521063:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00000800:00000001:2.0:1784948951.887535:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948951.887573:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948951.887585:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948951.887590:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948952.308093:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948952.308104:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948952.308112:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948952.308113:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948952.308191:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948952.308198:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948952.308203:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948953.332058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948953.332068:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948953.332076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948953.332078:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948953.332221:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948953.332228:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948953.332233:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948954.356034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948954.356044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948954.356053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948954.356055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948954.356177:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948954.356184:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948954.356188:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948955.380055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948955.380066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948955.380074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948955.380076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948955.380216:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948955.380225:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948955.380231:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948956.404025:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948956.404035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948956.404044:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948956.404046:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948956.404170:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948956.404177:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948956.404182:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948956.596102:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948956.596119:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948956.596121:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948956.596128:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948956.596132:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948956.596133:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948956.596140:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948956.596144:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948956.596146:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948956.596155:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007bd58f9d. 00000020:00000040:1.0:1784948956.596165:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948956.596170:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948956.596175:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948956.596183:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948956.596193:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000081367938. 00000100:00000001:1.0:1784948956.596202:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948956.596213:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948956.596228:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948956.596231:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948956.596237:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bd58f9d] to pc [ptlrpcd_00_03+3] req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948956.596283:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948956.596288:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948956.596294:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948956.596298:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948956.596303:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948956.596309:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000069457997. 00000100:00000001:2.0:1784948956.596332:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948956.596337:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1784948956.596341:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948956.596341:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948956.596350:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948956.596355:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1784948956.596356:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948956.596363:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948956.596376:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000adf08f9b. 00000100:00000001:2.0:1784948956.596381:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948956.596386:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948956.596387:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948956.596400:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784948956.596405:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1784948956.596406:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948956.596417:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948956.596431:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289664384:0@lo:400: 00000100:00000001:1.0:1784948956.596436:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948956.596441:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948956.596443:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784948956.596451:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069457997] to pc [ptlrpcd_00_00+0] req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948956.596453:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948956.596460:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948956.596467:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784948956.596477:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 02000000:00000001:2.0:1784948956.596484:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948956.596494:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000919ac10c. 00000100:00000001:1.0:1784948956.596500:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948956.596506:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000100:00000040:1.0:1784948956.596510:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11233) 00000100:00000200:2.0:1784948956.596519:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289664384, portal 25 00000100:00000001:2.0:1784948956.596526:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948956.596528:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549161600 : -115544160390016 : ffff96e9c75d3480) 00000100:00000040:2.0:1784948956.596537:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948956.596548:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948956.596552:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784948956.596554:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784948956.596560:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000100:00000001:0.0:1784948956.596561:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948956.596561:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948956.596569:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289664384, offset 0 00000100:00000001:0.0:1784948956.596574:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948956.596581:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784948956.596583:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948956.596596:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948956.596598:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948956.596602:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948956.596611:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948956.596634:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1784948956.596638:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438b80 00000100:00000001:0.0:1784948956.596641:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948956.596651:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784948956.596653:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7f7d [2] + 0 02000000:00000001:0.0:1784948956.596659:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784948956.596661:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002e5b8264. 00000400:00000200:2.0:1784948956.596670:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000037ca0070 02000000:00000001:0.0:1784948956.596672:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948956.596682:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:0.0:1784948956.596685:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289664512:192.168.202.137@tcp:400: 00000400:00000200:2.0:1784948956.596694:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948956.596705:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1784948956.596708:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000037ca0070 02000000:00000001:0.0:1784948956.596713:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:2.0:1784948956.596714:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000037ca0070. 02000000:00000001:0.0:1784948956.596723:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948956.596725:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948956.596730:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:0.0:1784948956.596734:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784948956.596737:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000d416d8c1 x1871651289664384 msgsize 224 02000000:00000010:0.0:1784948956.596744:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 00000100:00100000:2.0:1784948956.596752:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 02000000:00000001:0.0:1784948956.596755:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948956.596757:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000010:0.0:1784948956.596771:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009b268f62. 00000400:00000010:0.0:1784948956.596780:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000100:00000001:2.0:1784948956.596786:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:0.0:1784948956.596788:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289664512, portal 10 00000100:00000001:0.0:1784948956.596796:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:2.0:1784948956.596798:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000100:00000001:0.0:1784948956.596800:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549155328 : -115544160396288 : ffff96e9c75d1c00) 00000400:00000010:2.0:1784948956.596804:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000100:00000001:2.0:1784948956.596813:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784948956.596822:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948956.596823:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948956.596846:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948956.596854:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289664384 02000000:00000001:1.0:1784948956.596860:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948956.596867:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948956.596873:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948956.596883:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948956.596894:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289664384 00000100:00000001:0.0:1784948956.596897:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784948956.596899:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784948956.596901:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948956.596937:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784948956.596941:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948956.596947:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000040:0.0:1784948956.596953:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784948956.596954:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784948956.596963:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000010:0.0:1784948956.596965:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00000040:2.0:1784948956.596966:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948956.596973:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:0.0:1784948956.596977:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289664512, offset 0 00000020:00000010:1.0:1784948956.596984:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f7efedb6. 00000100:00000001:1.0:1784948956.596996:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948956.596998:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948956.597003:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f7efedb6. 00000100:00000001:2.0:1784948956.597008:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948956.597009:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000010:1.0:1784948956.597019:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 00000100:00000001:2.0:1784948956.597023:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948956.597029:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006e1d5987. 00000100:00000001:2.0:1784948956.597032:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305895979 00000400:00000200:0.0:1784948956.597039:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000010:1.0:1784948956.597040:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000010:2.0:1784948956.597044:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 00000100:00000040:1.0:1784948956.597058:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:0.0:1784948956.597060:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784948956.597061:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784948956.597065:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948956.597072:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948956.597079:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305895979 00000100:00000001:1.0:1784948956.597080:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948956.597081:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000010:2.0:1784948956.597094:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f59314ac. 00000100:00000001:1.0:1784948956.597099:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948956.597101:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784948956.597104:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948956.597117:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305895979 00000400:00000200:0.0:1784948956.597119:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784948956.597121:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784948956.597126:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000082553b41. 00000100:00000001:2.0:1784948956.597136:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000400:00000200:0.0:1784948956.597139:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3434:3434:241:4294967295] 192.168.202.137@tcp LPNI seq info [2218:2218:8:4294967295] 00000100:00000001:2.0:1784948956.597147:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305895979 00000100:00000001:1.0:1784948956.597151:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000010:2.0:1784948956.597154:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000abbd7d53. 00000100:00000001:1.0:1784948956.597157:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948956.597162:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00100000:1.0:1784948956.597172:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 743 00000100:00000001:2.0:1784948956.597177:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948956.597179:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:0.0:1784948956.597187:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784948956.597190:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466828880 : -115544242722736 : ffff96e9c274e850) 00000100:00000001:2.0:1784948956.597191:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948956.597204:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948956.597208:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d416d8c1 x1871651289664384/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:617/0 lens 224/0 e 0 to 0 dl 1784948967 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948956.597214:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948956.597218:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948956.597227:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948956.597233:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000010:0.0:1784948956.597234:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f64a2f6c. 00000100:00000001:2.0:1784948956.597243:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948956.597248:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948956.597250:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948956.597254:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948956.597261:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f7efedb6. 00000800:00000200:0.0:1784948956.597262:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784948956.597263:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948956.597273:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948956.597279:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f7efedb6. 00000800:00000200:0.0:1784948956.597294:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:1.0:1784948956.597295:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d416d8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289664384:12345-0@lo:400: 00000100:00000200:1.0:1784948956.597310:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289664384 00000800:00000200:0.0:1784948956.597311:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f64a2f6c type 1, nob 320 niov 1 nkiov 1 00000020:00000001:1.0:1784948956.597342:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948956.597351:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948956.597355:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948956.597360:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948956.597364:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784948956.597368:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948956.597370:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948956.597376:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948956.597381:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2113 usecs (3 jiffies) 00000020:00000001:1.0:1784948956.597382:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948956.597388:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948956.597393:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.597395:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305895940 00000020:00000001:1.0:1784948956.597397:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948956.597404:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948956.597407:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305895979 02000000:00000001:1.0:1784948956.597410:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784948956.597418:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305895940 02000000:00000010:1.0:1784948956.597419:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a20b4aa7. 00000100:00000001:0.0:1784948956.597426:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305895979 02000000:00000001:1.0:1784948956.597427:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948956.597432:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.597440:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305895940 00000020:00000001:1.0:1784948956.597441:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948956.597447:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289664384 00000100:00000001:0.0:1784948956.597453:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305895979 00010000:00000001:1.0:1784948956.597454:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948956.597463:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305895940 00010000:00000200:1.0:1784948956.597464:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d416d8c1 x1871651289664384/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:617/0 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948956.597475:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305895979 00010000:00000001:1.0:1784948956.597484:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784948956.597488:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1784948956.597489:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948956.597499:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=128 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948956.597500:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948956.597513:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784948956.597515:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948956.597521:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784948956.597527:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948956.597527:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948956.597538:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948956.597542:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.597545:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948956.597546:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948956.597554:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:1.0:1784948956.597555:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948956.597562:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000001:0.0:1784948956.597565:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784948956.597571:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289664384, offset 224 00000100:00000001:0.0:1784948956.597572:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948956.597580:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948956.597588:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948956.597591:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948956.597603:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948956.597605:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784948956.597624:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438b80 00000800:00000001:3.0:1784948956.597627:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948956.597636:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948956.597637:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x891d [1] + 224 00000800:00000001:3.0:1784948956.597648:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948956.597654:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948956.597672:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948956.597673:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948956.597690:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948956.597701:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948956.597707:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438c00 00000400:00000200:3.0:1784948956.597728:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 162096 00000100:00000040:1.0:1784948956.597747:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948956.597749:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948956.597758:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948956.597768:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f64a2f6c. 00000800:00000001:3.0:1784948956.597770:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948956.597778:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948956.597779:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948956.597783:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1784948956.597799:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948956.597801:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784948956.597808:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:1.0:1784948956.597814:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948956.597817:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784948956.597818:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784948956.597824:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948956.597826:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:1.0:1784948956.597839:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000010:3.0:1784948956.597840:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000048dc6bdd. 00010000:00000001:1.0:1784948956.597846:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948956.597853:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948956.597854:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000048dc6bdd x1871651289664512 msgsize 224 00000100:00000001:0.0:1784948956.597857:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784948956.597858:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784948956.597866:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948956.597873:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d416d8c1 x1871651289664384/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:617/0 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948956.597873:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000001:0.0:1784948956.597879:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948956.597887:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1784948956.597889:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000010:0.0:1784948956.597895:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000100:00100000:1.0:1784948956.597942:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d416d8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289664384:12345-0@lo:400: Request processed in 691us (1202us total) trans 0 rc 0/0 00000400:00000200:0.0:1784948956.597944:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000100:00000001:3.0:1784948956.597946:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:0.0:1784948956.597953:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000100:00100000:1.0:1784948956.597962:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 743 00000100:00000001:0.0:1784948956.597966:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784948956.597968:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:2.0:1784948956.597968:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:2.0:1784948956.597978:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000100:00000001:1.0:1784948956.597979:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:0.0:1784948956.597980:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948956.597985:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784948956.597991:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1784948956.597996:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006e1d5987. 00000800:00000001:3.0:1784948956.598002:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:2.0:1784948956.598005:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948956.598008:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000800:00000001:3.0:1784948956.598012:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948956.598020:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00000100:00000200:0.0:1784948956.598029:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784948956.598033:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:1.0:1784948956.598045:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784948956.598051:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a20b4aa7. 00000100:00000001:2.0:1784948956.598052:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784948956.598063:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000040:2.0:1784948956.598068:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948956.598073:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.598074:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948956.598085:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784948956.598087:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948956.598090:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.598097:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:2.0:1784948956.598098:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:1.0:1784948956.598104:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000acd175a6. 02000000:00000001:0.0:1784948956.598106:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948956.598114:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:1.0:1784948956.598114:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000575a8ab. 00000100:00000001:0.0:1784948956.598122:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948956.598130:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948956.598138:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=128 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948956.598147:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=128 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948956.598154:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948956.598158:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948956.598164:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948956.598168:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948956.598172:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784948956.598174:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948956.598177:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289664512 02000000:00000001:3.0:1784948956.598181:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:0.0:1784948956.598183:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948956.598184:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948956.598187:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948956.598193:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.598199:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784948956.598199:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289664512 00000020:00000001:3.0:1784948956.598204:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948956.598205:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:3.0:1784948956.598207:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784948956.598208:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948956.598212:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784948956.598216:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784948956.598217:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948956.598223:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784948956.598229:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00100000:0.0:1784948956.598232:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289664384:0@lo:400: 00000100:00000001:3.0:1784948956.598235:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948956.598243:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948956.598244:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:0.0:1784948956.598247:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948956.598249:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948956.598252:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007bd58f9d x1871651289664384/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948956.598253:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000020:00000010:3.0:1784948956.598261:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003f3c01c8. 00000100:00000001:0.0:1784948956.598267:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000010:3.0:1784948956.598268:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000001ceb335. 02000000:00000001:0.0:1784948956.598271:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948956.598274:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 00000020:00000010:3.0:1784948956.598275:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 02000000:00000001:0.0:1784948956.598280:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948956.598284:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948956.598286:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000040:3.0:1784948956.598287:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:0.0:1784948956.598291:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784948956.598291:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948956.598294:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000010:0.0:1784948956.598295:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000081367938. 00000100:00000001:3.0:1784948956.598298:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948956.598301:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948956.598305:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1784948956.598305:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000010:0.0:1784948956.598308:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007bd58f9d. 00000100:00000001:3.0:1784948956.598309:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948956.598341:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.598343:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948956.598346:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.598349:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.598357:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948956.598359:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948956.598362:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.598363:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948956.598367:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948956.598379:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948956.598382:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948956.598392:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 719 00000100:00000040:3.0:1784948956.598398:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948956.598403:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554948096 : -115544154603520 : ffff96e9c7b58000) 00000100:00000040:3.0:1784948956.598413:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000048dc6bdd x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:617/0 lens 224/0 e 0 to 0 dl 1784948967 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948956.598431:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948956.598436:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:3.0:1784948956.598441:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948956.598445:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000100:00100000:3.0:1784948956.598452:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000048dc6bdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289664512:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784948956.598456:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289664512 00000020:00000001:3.0:1784948956.598459:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948956.598463:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948956.598465:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948956.598468:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948956.598470:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784948956.598474:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948956.598478:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948956.598480:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948956.598482:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948956.598485:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948956.598487:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948956.598491:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948956.598494:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948956.598498:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000fd4fe1c8. 02000000:00000001:3.0:1784948956.598500:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948956.598502:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948956.598505:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948956.598507:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289664512 00010000:00000001:3.0:1784948956.598510:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948956.598514:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000048dc6bdd x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:617/0 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948956.598523:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948956.598526:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948956.598528:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=123 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948956.598532:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948956.598535:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948956.598538:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784948956.598541:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948956.598543:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948956.598545:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948956.598548:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948956.598552:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000100:00000200:3.0:1784948956.598556:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289664512, offset 224 00000400:00000200:3.0:1784948956.598561:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948956.598570:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948956.598576:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3435:3435:241:4294967295] 192.168.202.137@tcp LPNI seq info [2219:2219:8:4294967295] 00000400:00000200:3.0:1784948956.598588:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948956.598593:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948956.598598:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784948956.598604:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948956.598610:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948956.598614:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948956.598632:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948956.598636:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948956.598638:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948956.598640:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948956.598643:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948956.598647:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000048dc6bdd x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:617/0 lens 224/224 e 0 to 0 dl 1784948967 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948956.598657:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000048dc6bdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289664512:12345-192.168.202.137@tcp:400: Request processed in 226us (803us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948956.598666:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 719 00000100:00000040:3.0:1784948956.598670:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948956.598673:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948956.598676:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948956.598680:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000001ceb335. 00000020:00000010:3.0:1784948956.598685:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784948956.598689:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000003f3c01c8. 00000020:00000040:3.0:1784948956.598695:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948956.598699:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948956.598760:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948956.598767:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:2.0:1784948956.598776:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948956.598779:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948956.598786:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948956.598793:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948956.598794:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:2.0:1784948956.598801:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000800:00000001:3.0:1784948956.598803:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948956.598808:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784948956.598810:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948956.598813:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948956.598825:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948956.598837:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438c00 00000400:00000200:3.0:1784948956.598845:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8925 [1] + 224 00000800:00000001:3.0:1784948956.598852:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948956.598858:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948956.598860:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948956.598866:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948956.598871:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948956.598875:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948956.598886:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948956.598940:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948956.598952:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948956.598958:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.598974:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948956.598978:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948956.598984:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948956.598986:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948956.598990:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009b268f62. 00000400:00000200:0.0:1784948956.598997:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000400:00000010:0.0:1784948956.599000:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000001:0.0:1784948956.599006:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948956.599013:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948956.599030:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948956.599041:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948956.599043:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.599045:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948956.599048:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948956.599050:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948956.599052:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948956.599055:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948956.599061:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=123 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948956.599065:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=123 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948956.599068:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948956.599069:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948956.599072:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948956.599075:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948956.599084:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.599088:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948956.599090:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948956.599093:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948956.599094:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948956.599098:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948956.599106:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948956.599108:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948956.599109:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948956.599112:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948956.599121:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289664512:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948956.599126:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.599128:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948956.599131:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000069457997 x1871651289664512/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948956.599139:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948956.599140:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948956.599142:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 02000000:00000001:0.0:1784948956.599146:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948956.599148:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948956.599149:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948956.599151:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948956.599153:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000adf08f9b. 02000000:00000001:0.0:1784948956.599156:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948956.599158:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948956.599159:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000069457997. 00000100:00000001:0.0:1784948956.599163:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948956.599164:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.599166:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948956.599171:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948956.599173:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948956.599175:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948956.599178:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948956.639982:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305895979, at 4305895980 00000100:00000010:2.0:1784948956.639996:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000100:00000040:1.0:1784948956.640032:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305895979, at 4305895980 00000100:00000010:1.0:1784948956.640046:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f59314ac. 00000100:00000040:0.0:1784948956.640058:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305895979, at 4305895980 00000100:00000040:3.0:1784948956.640058:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305895979, at 4305895980 00000100:00000010:3.0:1784948956.640077:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000abbd7d53. 00000100:00000010:0.0:1784948956.640078:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00010000:00000001:2.0:1784948957.428026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948957.428035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948957.428043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948957.428055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948957.428176:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948957.428182:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948957.428187:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948957.620023:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948957.620037:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948957.620039:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948957.620045:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948957.620048:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948957.620052:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948957.620055:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948958.452034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948958.452044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948958.452052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948958.452054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948958.452186:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948958.452193:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948958.452198:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948958.644016:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948958.644043:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948958.644046:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948958.644054:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948958.644057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948958.644062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948958.644065:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948959.475975:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948959.475982:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948959.475988:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948959.475989:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948959.476027:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948959.476030:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948959.476033:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948959.668012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948959.668026:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948959.668029:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948959.668035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948959.668038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948959.668042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948959.668045:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948960.499993:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948960.500000:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948960.500005:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948960.500008:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948960.500011:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948960.500014:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948960.500016:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948960.500018:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948960.500019:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948960.500021:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948960.500077:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948960.500082:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948960.500086:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948960.692011:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948960.692027:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948960.692029:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948960.692035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948960.692039:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948960.692043:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948960.692046:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948961.524047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948961.524057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948961.524065:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948961.524067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948961.524208:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948961.524215:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948961.524219:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948961.716045:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948961.716064:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948961.716067:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948961.716069:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948961.716078:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948961.716079:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948961.716085:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948961.716091:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948961.716092:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a7f2082b. 00000100:00000001:3.0:1784948961.716097:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948961.716102:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948961.716109:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948961.716113:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948961.716119:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948961.716128:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f64a2f6c. 00000100:00000001:0.0:1784948961.716135:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948961.716145:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948961.716157:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948961.716160:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948961.716165:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7f2082b] to pc [ptlrpcd_00_01+1] req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948961.716195:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948961.716198:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948961.716202:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948961.716206:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948961.716210:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948961.716213:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000061b68cb4. 00000020:00000040:0.0:1784948961.716217:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948961.716220:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948961.716222:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948961.716223:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948961.716228:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784948961.716229:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000000d1fbdb0. 00000100:00000001:1.0:1784948961.716229:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948961.716237:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.716238:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948961.716238:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948961.716244:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948961.716253:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1784948961.716254:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948961.716296:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948961.716316:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948961.716326:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061b68cb4] to pc [ptlrpcd_00_02+2] req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948961.716334:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948961.716340:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948961.716349:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948961.716353:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948961.716357:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948961.716362:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a7f2082b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289664640:0@lo:400: 00000100:00000001:1.0:1784948961.716368:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948961.716375:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784948961.716378:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948961.716379:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948961.716388:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784948961.716391:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11238) 02000000:00000010:1.0:1784948961.716396:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000378c7327. 02000000:00000001:1.0:1784948961.716402:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.716410:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784948961.716411:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e572277. 00000100:00100000:0.0:1784948961.716419:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:3.0:1784948961.716421:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948961.716425:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784948961.716425:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000001:3.0:1784948961.716429:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.716439:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:1.0:1784948961.716440:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289664640, portal 25 00000100:00000001:0.0:1784948961.716445:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948961.716447:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948961.716451:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948961.716452:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554965376 : -115544154586240 : ffff96e9c7b5c380) 00000100:00000001:0.0:1784948961.716456:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948961.716461:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948961.716465:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948961.716472:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948961.716497:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784948961.716503:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784948961.716510:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948961.716511:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948961.716518:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 02000000:00000001:0.0:1784948961.716519:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948961.716524:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1784948961.716526:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289664640, offset 0 02000000:00000001:0.0:1784948961.716531:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948961.716537:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000061b68cb4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289664768:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784948961.716547:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948961.716548:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948961.716572:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948961.716575:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948961.716577:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948961.716583:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 00000400:00000200:1.0:1784948961.716586:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:0.0:1784948961.716588:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948961.716596:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009b268f62. 00000400:00000010:0.0:1784948961.716602:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000400:00000200:1.0:1784948961.716604:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438c80 00000100:00000200:0.0:1784948961.716609:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289664768, portal 10 00000100:00000001:0.0:1784948961.716614:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784948961.716616:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7f95 [2] + 0 00000100:00000001:0.0:1784948961.716619:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554974336 : -115544154577280 : ffff96e9c7b5e680) 00000400:00000010:1.0:1784948961.716624:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000930107ef. 00000100:00000040:0.0:1784948961.716632:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948961.716633:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000ed7b5cc8 00000100:00000001:0.0:1784948961.716649:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784948961.716649:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784948961.716659:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948961.716665:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000010:0.0:1784948961.716668:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00000400:00000200:1.0:1784948961.716674:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed7b5cc8 00000400:00000010:1.0:1784948961.716680:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000ed7b5cc8. 00000100:00000200:0.0:1784948961.716688:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289664768, offset 0 00000100:00000001:1.0:1784948961.716688:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948961.716694:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784948961.716701:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000ab8619ba x1871651289664640 msgsize 224 00000400:00000200:0.0:1784948961.716704:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784948961.716717:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:0.0:1784948961.716722:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:1.0:1784948961.716724:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784948961.716730:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784948961.716736:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784948961.716740:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948961.716744:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784948961.716748:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784948961.716749:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3436:3436:241:4294967295] 192.168.202.137@tcp LPNI seq info [2220:2220:8:4294967295] 00000400:00000200:0.0:1784948961.716766:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948961.716772:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948961.716778:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000400:00000200:1.0:1784948961.716780:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784948961.716785:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:2.0:1784948961.716788:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000200:0.0:1784948961.716791:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948961.716793:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:2.0:1784948961.716795:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289664640 02000000:00000001:2.0:1784948961.716801:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000200:1.0:1784948961.716804:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948961.716806:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:0.0:1784948961.716812:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784948961.716813:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948961.716823:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948961.716824:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948961.716829:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1784948961.716832:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289664640 00000100:00000040:1.0:1784948961.716836:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948961.716837:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948961.716841:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948961.716844:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948961.716844:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948961.716850:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000040:0.0:1784948961.716854:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2115 usecs (3 jiffies) 00000020:00000001:2.0:1784948961.716859:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948961.716864:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305901059 00000020:00000001:2.0:1784948961.716868:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:0.0:1784948961.716875:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:2.0:1784948961.716878:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784948961.716884:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948961.716888:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:2.0:1784948961.716888:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:0.0:1784948961.716889:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784948961.716899:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948961.716901:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305901059 00000100:00000001:1.0:1784948961.716902:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948961.716938:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305901099 00000020:00000010:2.0:1784948961.716941:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000010:0.0:1784948961.716943:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:1.0:1784948961.716948:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305901059 00000100:00000001:0.0:1784948961.716951:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000010:2.0:1784948961.716953:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000075318cf7. 00000100:00000001:1.0:1784948961.716957:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305901099 00000100:00000001:0.0:1784948961.716962:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305901060 00000020:00000010:2.0:1784948961.716963:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000091201221. 00000100:00000001:1.0:1784948961.716965:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305901059 00000100:00000010:0.0:1784948961.716974:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:1.0:1784948961.716976:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305901099 00000020:00000010:2.0:1784948961.716976:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:0.0:1784948961.716983:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948961.716992:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305901060 00000100:00000040:2.0:1784948961.716995:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948961.716996:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305901060 00000100:00000010:0.0:1784948961.716997:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:2.0:1784948961.717003:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784948961.717007:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948961.717008:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948961.717011:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305901099 00000100:00000001:2.0:1784948961.717014:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717019:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948961.717023:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305901060 00000100:00000001:0.0:1784948961.717024:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948961.717026:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948961.717033:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948961.717034:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948961.717041:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948961.717043:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717045:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948961.717055:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948961.717059:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948961.717067:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948961.717067:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948961.717069:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784948961.717073:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948961.717076:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948961.717079:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1784948961.717084:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 744 00000100:00000001:1.0:1784948961.717085:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948961.717091:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948961.717092:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948961.717097:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466826832 : -115544242724784 : ffff96e9c274e050) 00000100:00000040:2.0:1784948961.717107:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ab8619ba x1871651289664640/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:622/0 lens 224/0 e 0 to 0 dl 1784948972 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948961.717117:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948961.717122:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:2.0:1784948961.717126:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948961.717129:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00100000:2.0:1784948961.717137:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ab8619ba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289664640:12345-0@lo:400: 00000100:00000200:2.0:1784948961.717142:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289664640 00000020:00000001:2.0:1784948961.717145:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948961.717149:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948961.717151:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948961.717154:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948961.717156:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784948961.717159:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948961.717161:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948961.717164:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948961.717166:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784948961.717169:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948961.717172:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948961.717174:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948961.717177:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784948961.717181:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c38711c7. 02000000:00000001:2.0:1784948961.717184:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948961.717186:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948961.717189:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948961.717192:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289664640 00010000:00000001:2.0:1784948961.717195:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784948961.717200:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ab8619ba x1871651289664640/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:622/0 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948961.717210:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784948961.717212:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948961.717217:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=133 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948961.717221:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948961.717224:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948961.717226:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784948961.717229:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948961.717231:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948961.717234:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948961.717237:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948961.717242:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000100:00000200:2.0:1784948961.717246:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289664640, offset 224 00000400:00000200:2.0:1784948961.717251:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784948961.717259:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784948961.717268:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438c80 00000400:00000200:2.0:1784948961.717319:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8939 [1] + 224 00000400:00000200:2.0:1784948961.717331:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948961.717336:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948961.717344:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000200:3.0:1784948961.717347:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000200:2.0:1784948961.717354:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1784948961.717358:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000800:00000001:1.0:1784948961.717370:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948961.717373:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784948961.717383:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948961.717388:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948961.717392:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948961.717398:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948961.717406:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948961.717410:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000400:00000010:3.0:1784948961.717420:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000400:00000200:1.0:1784948961.717428:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948961.717433:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948961.717434:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1784948961.717443:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438d00 00000400:00000200:2.0:1784948961.717447:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000100:00000200:3.0:1784948961.717450:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948961.717455:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000100:00000001:2.0:1784948961.717463:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948961.717467:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948961.717473:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 162320 00000100:00000001:2.0:1784948961.717479:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948961.717492:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784948961.717494:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784948961.717496:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784948961.717497:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948961.717504:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948961.717511:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948961.717511:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784948961.717512:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948961.717520:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784948961.717525:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ab8619ba x1871651289664640/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:622/0 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948961.717539:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948961.717571:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948961.717576:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784948961.717579:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948961.717582:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717585:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1784948961.717587:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000096f3921c. 00000100:00000001:3.0:1784948961.717591:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784948961.717593:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ab8619ba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289664640:12345-0@lo:400: Request processed in 470us (885us total) trans 0 rc 0/0 00000400:00000001:0.0:1784948961.717596:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1784948961.717597:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000096f3921c x1871651289664768 msgsize 224 00000400:00000001:0.0:1784948961.717602:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948961.717607:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e572277. 00000100:00100000:2.0:1784948961.717616:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 744 00000100:00100000:1.0:1784948961.717617:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:0.0:1784948961.717618:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000100:00000040:2.0:1784948961.717624:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000010:0.0:1784948961.717626:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:2.0:1784948961.717630:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948961.717634:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784948961.717635:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784948961.717636:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:2.0:1784948961.717643:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000091201221. 00000800:00000001:1.0:1784948961.717648:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784948961.717649:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784948961.717651:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000800:00000001:1.0:1784948961.717655:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948961.717660:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000075318cf7. 00000020:00000040:2.0:1784948961.717669:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948961.717674:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:2.0:1784948961.717678:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000200:0.0:1784948961.717681:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948961.717681:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289664768 02000000:00000010:2.0:1784948961.717682:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c38711c7. 02000000:00000001:3.0:1784948961.717690:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:2.0:1784948961.717693:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948961.717697:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948961.717698:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948961.717705:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717709:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:2.0:1784948961.717710:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:3.0:1784948961.717714:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717716:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948961.717718:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004a584a71. 00000100:00100000:3.0:1784948961.717724:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289664768 00000100:00000001:0.0:1784948961.717725:0:337120:0:(client.c:1580:after_reply()) Process entered 00000400:00000010:2.0:1784948961.717727:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d5e12ff0. 02000000:00000001:0.0:1784948961.717731:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:3.0:1784948961.717733:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948961.717735:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:3.0:1784948961.717739:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784948961.717740:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948961.717746:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948961.717748:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00001000:0.0:1784948961.717752:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=133 v=5 (5 5 5 5) 00000020:00000040:3.0:1784948961.717755:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00001000:0.0:1784948961.717759:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=133 v=5 (1 1 1 1) 00000020:00000001:3.0:1784948961.717765:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784948961.717766:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948961.717769:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948961.717775:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:0.0:1784948961.717776:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784948961.717779:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717785:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948961.717788:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:0.0:1784948961.717792:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948961.717797:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000001:0.0:1784948961.717807:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948961.717808:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948961.717813:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948961.717819:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784948961.717819:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000100:00000040:0.0:1784948961.717828:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948961.717833:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000100:00100000:0.0:1784948961.717838:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a7f2082b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289664640:0@lo:400: 00000020:00000010:3.0:1784948961.717844:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c6655d60. 00000100:00000001:0.0:1784948961.717845:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948961.717849:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948961.717853:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a7f2082b x1871651289664640/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784948961.717855:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784948961.717866:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948961.717869:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948961.717871:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000378c7327. 00000100:00000040:3.0:1784948961.717873:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:0.0:1784948961.717876:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1784948961.717880:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000001:0.0:1784948961.717881:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948961.717885:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948961.717887:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:0.0:1784948961.717889:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948961.717894:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f64a2f6c. 00000100:00000001:3.0:1784948961.717896:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784948961.717898:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948961.717902:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948961.717905:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a7f2082b. 00000100:00000001:0.0:1784948961.717916:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948961.717919:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948961.717948:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948961.717953:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948961.717956:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717960:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948961.717968:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948961.717970:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948961.717972:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948961.717974:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948961.717980:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948961.717998:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948961.718002:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948961.718011:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 720 00000100:00000040:3.0:1784948961.718016:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948961.718019:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549153536 : -115544160398080 : ffff96e9c75d1500) 00000100:00000040:3.0:1784948961.718029:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000096f3921c x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:622/0 lens 224/0 e 0 to 0 dl 1784948972 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948961.718043:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948961.718048:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000001:3.0:1784948961.718052:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948961.718054:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000100:00100000:3.0:1784948961.718061:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000096f3921c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289664768:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784948961.718067:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289664768 00000020:00000001:3.0:1784948961.718070:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948961.718073:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948961.718076:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948961.718079:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948961.718082:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784948961.718103:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948961.718107:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948961.718111:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784948961.718113:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948961.718116:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948961.718119:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948961.718124:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948961.718127:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948961.718132:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000df952d52. 02000000:00000001:3.0:1784948961.718137:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948961.718140:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948961.718143:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948961.718147:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289664768 00010000:00000001:3.0:1784948961.718151:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948961.718157:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000096f3921c x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:622/0 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948961.718171:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948961.718173:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948961.718177:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=128 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948961.718183:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948961.718187:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948961.718191:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784948961.718195:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948961.718197:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948961.718200:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948961.718205:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948961.718210:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000100:00000200:3.0:1784948961.718216:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289664768, offset 224 00000400:00000200:3.0:1784948961.718225:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948961.718238:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948961.718245:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3437:3437:241:4294967295] 192.168.202.137@tcp LPNI seq info [2221:2221:8:4294967295] 00000400:00000200:3.0:1784948961.718261:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948961.718294:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948961.718318:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784948961.718326:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948961.718334:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948961.718338:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948961.718356:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948961.718361:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948961.718365:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948961.718384:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948961.718387:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948961.718394:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000096f3921c x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:622/0 lens 224/224 e 0 to 0 dl 1784948972 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948961.718411:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000096f3921c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289664768:12345-192.168.202.137@tcp:400: Request processed in 365us (811us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948961.718425:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 720 00000100:00000040:3.0:1784948961.718431:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948961.718435:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948961.718438:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948961.718442:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c6655d60. 00000020:00000010:3.0:1784948961.718449:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784948961.718455:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 00000020:00000040:3.0:1784948961.718466:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948961.718470:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948961.718514:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948961.718519:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:0.0:1784948961.718524:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948961.718530:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948961.718534:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:0.0:1784948961.718538:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000100:00000001:0.0:1784948961.718542:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948961.718544:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948961.718552:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948961.718574:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948961.718576:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948961.718579:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948961.718584:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948961.718592:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438d00 00000400:00000200:0.0:1784948961.718597:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8941 [1] + 224 00000800:00000001:0.0:1784948961.718602:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948961.718606:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948961.718608:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948961.718611:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948961.718614:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948961.718617:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948961.718627:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948961.718653:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948961.718661:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948961.718665:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948961.718679:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948961.718684:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948961.718691:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948961.718693:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948961.718695:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009b268f62. 00000400:00000200:2.0:1784948961.718701:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000400:00000010:2.0:1784948961.718703:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000001:2.0:1784948961.718706:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948961.718712:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948961.718722:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948961.718729:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948961.718731:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948961.718733:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948961.718735:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948961.718737:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948961.718739:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948961.718741:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948961.718745:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=128 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948961.718748:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=128 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948961.718751:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948961.718752:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948961.718754:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948961.718757:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948961.718764:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948961.718767:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948961.718768:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948961.718770:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948961.718772:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948961.718774:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948961.718780:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948961.718782:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948961.718783:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948961.718785:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948961.718792:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000061b68cb4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289664768:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948961.718795:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948961.718797:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948961.718799:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000061b68cb4 x1871651289664768/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948961.718805:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948961.718807:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948961.718808:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 02000000:00000001:2.0:1784948961.718812:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948961.718813:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948961.718814:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948961.718817:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948961.718818:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000000d1fbdb0. 02000000:00000001:2.0:1784948961.718821:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948961.718822:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948961.718823:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000061b68cb4. 00000100:00000001:2.0:1784948961.718834:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948961.718835:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948961.718837:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948961.718841:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948961.718842:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948961.718844:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948961.718846:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948961.759995:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305901099, at 4305901100 00000100:00000010:1.0:1784948961.760009:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:3.0:1784948961.760021:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305901099, at 4305901100 00000100:00000010:3.0:1784948961.760034:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000040:2.0:1784948961.760045:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305901099, at 4305901100 00000100:00000040:0.0:1784948961.760046:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305901099, at 4305901100 00000100:00000010:2.0:1784948961.760059:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:0.0:1784948961.760064:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00010000:00000001:2.0:1784948962.547994:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948962.548002:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948962.548008:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948962.548009:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948962.548077:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948962.548082:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948962.548085:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948962.740058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948962.740075:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948962.740078:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948962.740087:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948962.740091:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948962.740097:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948962.740102:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784948962.932068:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.10@tcp 00000800:00000010:1.0:1784948962.932075:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000662152ce. 00000400:00000200:1.0:1784948962.932086:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948962.932093:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948962.932101:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11235 health :900 00000400:00000200:1.0:1784948962.932108:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948962.932113:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:1.0:1784948962.932118:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:2.0:1784948962.932247:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948962.932285:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000001:2.0:1784948962.932295:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948962.932297:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948962.932301:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:2.0:1784948962.932305:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:2.0:1784948962.932310:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948962.932315:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:2.0:1784948962.932321:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef171f31. 00000400:00000200:2.0:1784948962.932328:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-110 00000400:00000200:2.0:1784948962.932331:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -110 00000400:00000200:2.0:1784948962.932335:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00010000:00000001:2.0:1784948963.571983:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948963.571990:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948963.571996:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948963.571997:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948963.572076:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948963.572080:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948963.572082:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948963.764121:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948963.764136:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948963.764138:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948963.764143:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948963.764147:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948963.764151:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948963.764153:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948963.956026:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784948963.956035:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000000e755589. 00000400:00000010:3.0:1784948963.956046:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e5260462. 00000400:00000010:3.0:1784948963.956054:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 00000400:00000010:3.0:1784948963.956065:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000061508776. 00000800:00000100:1.0:1784948963.956072:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000400:00000200:3.0:1784948963.956082:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948963.956085:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:1.0:1784948963.956096:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948963.956108:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948963.956113:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948963.956124:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948963.956130:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3438:3438:242:4294967295] 192.168.202.137@tcp LPNI seq info [2222:2222:8:4294967295] 00000400:00000200:3.0:1784948963.956139:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948963.956148:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948963.956155:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948963.956162:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11236 health :800 00000800:00000010:3.0:1784948963.956164:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000400:00000200:1.0:1784948963.956173:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000200:3.0:1784948963.956182:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784948963.956184:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:1.0:1784948963.956195:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000800:00000200:3.0:1784948963.956202:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948963.956208:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784948963.956286:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948963.956300:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:2.0:1784948963.956309:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948963.956313:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948963.956316:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:2.0:1784948963.956320:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000400:00000200:2.0:1784948963.956326:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948963.956331:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:2.0:1784948963.956336:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003cdea007. 00000400:00000200:2.0:1784948963.956341:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:2.0:1784948963.956344:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:2.0:1784948963.956349:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00000800:00000200:0.0:1784948963.956436:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948963.956444:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000400:00000200:0.0:1784948963.956452:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948963.956459:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948963.956462:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948963.956471:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948963.956475:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784948963.956477:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948963.956484:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948963.956492:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948963.956502:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948963.956520:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948963.956530:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948963.956543:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948963.956548:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3439:3439:243:4294967295] 192.168.202.137@tcp LPNI seq info [2223:2223:8:4294967295] 00000400:00000200:2.0:1784948963.956570:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948963.956576:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948963.956581:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a934b7fa. 00000800:00000200:2.0:1784948963.956587:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948963.956594:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948963.956598:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a934b7fa type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948963.956680:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948963.956730:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948963.956736:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a934b7fa. 00000400:00000200:2.0:1784948963.956741:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948963.956747:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948963.956760:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948963.956765:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948963.956768:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948963.956776:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948963.956783:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948963.956790:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948963.956802:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8955 00000800:00000001:2.0:1784948963.956809:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948963.956816:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948963.956819:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948963.956825:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948963.956830:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096db69f7 00000400:00000010:2.0:1784948963.956832:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000096db69f7. 00000400:00000200:2.0:1784948963.956835:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948963.956838:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000000e755589. 00000400:00000010:2.0:1784948963.956842:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e5260462. 00000800:00000001:2.0:1784948963.956848:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948963.956851:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948964.596037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948964.596047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948964.596056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948964.596057:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948964.596144:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948964.596150:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948964.596154:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948964.788045:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948964.788059:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948964.788061:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948964.788068:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948964.788072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948964.788077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948964.788080:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784948964.980055:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000800:00000010:1.0:1784948964.980062:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006c136176. 00000400:00000200:1.0:1784948964.980076:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948964.980084:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948964.980092:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11238 health :900 00000400:00000200:1.0:1784948964.980100:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948964.980105:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:1.0:1784948964.980111:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:2.0:1784948964.980191:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948964.980203:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000001:2.0:1784948964.980213:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948964.980215:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948964.980218:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:2.0:1784948964.980222:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000400:00000200:2.0:1784948964.980227:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948964.980232:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:2.0:1784948964.980237:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 00000400:00000200:2.0:1784948964.980286:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:2.0:1784948964.980301:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:2.0:1784948964.980306:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00010000:00000001:2.0:1784948965.620038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948965.620047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948965.620056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948965.620058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948965.620180:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948965.620187:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948965.620191:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948965.811993:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948965.812008:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948965.812010:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948965.812016:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948965.812020:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948965.812024:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948965.812027:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948966.004075:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:1.0:1784948966.004079:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000400:00000010:3.0:1784948966.004087:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000f682530a. 00000800:00000010:1.0:1784948966.004088:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003a8a98bc. 00000400:00000200:1.0:1784948966.004101:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784948966.004103:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000041892d66. 00000400:00000200:1.0:1784948966.004113:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784948966.004114:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000400:00000200:1.0:1784948966.004128:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11239 health :800 00000400:00000200:1.0:1784948966.004137:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:3.0:1784948966.004141:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000570fe348. 00000400:00000200:1.0:1784948966.004143:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:3.0:1784948966.004157:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948966.004158:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:3.0:1784948966.004202:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948966.004213:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948966.004219:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3440:3440:244:4294967295] 192.168.202.137@tcp LPNI seq info [2224:2224:8:4294967295] 00000400:00000200:3.0:1784948966.004229:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948966.004273:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948966.004283:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948966.004291:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000400:00000200:2.0:1784948966.004293:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784948966.004304:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784948966.004311:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:2.0:1784948966.004322:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948966.004328:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:3.0:1784948966.004329:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:2.0:1784948966.004333:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 00000800:00000200:3.0:1784948966.004337:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 2, nob 96 niov 1 nkiov 0 00000400:00000010:2.0:1784948966.004339:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000400:00000200:2.0:1784948966.004350:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948966.004358:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:2.0:1784948966.004363:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000033c25eed. 00000400:00000200:2.0:1784948966.004372:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:2.0:1784948966.004376:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:2.0:1784948966.004383:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000800:00000200:0.0:1784948966.004589:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948966.004597:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000400:00000200:0.0:1784948966.004606:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948966.004616:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784948966.004618:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948966.004624:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948966.004630:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948966.004638:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948966.004645:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948966.004652:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948966.004665:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948966.004681:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948966.004692:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948966.004704:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948966.004710:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3441:3441:245:4294967295] 192.168.202.137@tcp LPNI seq info [2225:2225:8:4294967295] 00000400:00000200:1.0:1784948966.004722:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948966.004728:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948966.004733:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000800:00000200:1.0:1784948966.004739:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948966.004746:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948966.004750:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:1.0:1784948966.004872:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948966.004877:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:1.0:1784948966.004878:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000400:00000200:1.0:1784948966.004889:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948966.004891:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948966.004901:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948966.004901:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948966.004948:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948966.004953:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948966.004959:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948966.004972:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948966.004991:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8959 00000800:00000001:2.0:1784948966.005000:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948966.005012:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948966.005016:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948966.005020:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948966.005026:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:2.0:1784948966.005029:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000400:00000200:2.0:1784948966.005032:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948966.005036:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000f682530a. 00000400:00000010:2.0:1784948966.005039:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000041892d66. 00000800:00000001:2.0:1784948966.005047:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948966.005050:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948966.643990:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948966.643996:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948966.644017:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948966.644018:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948966.644073:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948966.644077:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948966.644079:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948966.836044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948966.836058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948966.836062:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948966.836068:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948966.836072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948966.836078:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948966.836080:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948966.836082:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948966.836094:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948966.836107:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a612e3ff. 00000020:00000040:1.0:1784948966.836116:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948966.836122:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948966.836126:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948966.836134:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948966.836144:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e79c6895. 00000100:00000001:1.0:1784948966.836153:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948966.836165:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948966.836180:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948966.836183:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948966.836190:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a612e3ff] to pc [ptlrpcd_00_03+3] req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948966.836279:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948966.836285:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948966.836291:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948966.836310:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948966.836313:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948966.836316:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb5afc66. 00000020:00000040:1.0:1784948966.836318:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948966.836321:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948966.836322:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948966.836325:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.836325:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948966.836330:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1784948966.836331:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000031deb17e. 00000100:00000001:2.0:1784948966.836331:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948966.836335:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.836336:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948966.836337:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948966.836347:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784948966.836347:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948966.836419:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948966.836421:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948966.836424:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.836427:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948966.836435:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784948966.836437:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb5afc66] to pc [ptlrpcd_00_00+0] req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948966.836442:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948966.836454:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948966.836468:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289664896:0@lo:400: 00000100:00000001:2.0:1784948966.836475:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948966.836483:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948966.836486:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948966.836489:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948966.836494:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948966.836498:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000075318cf7. 02000000:00000001:2.0:1784948966.836504:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948966.836505:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11243) 00000100:00000001:3.0:1784948966.836518:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784948966.836519:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000075224c9. 00000100:00000001:3.0:1784948966.836528:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948966.836532:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784948966.836533:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000492bb2ef. 00000100:00000001:3.0:1784948966.836534:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784948966.836541:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948966.836549:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289664896, portal 25 00000100:00000040:3.0:1784948966.836551:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948966.836560:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948966.836566:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549158016 : -115544160393600 : ffff96e9c75d2680) 00000100:00000040:2.0:1784948966.836581:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948966.836583:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948966.836590:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948966.836603:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784948966.836606:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784948966.836609:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948966.836618:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948966.836619:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00100000:3.0:1784948966.836630:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb5afc66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289665024:192.168.202.137@tcp:400: 00000400:00000010:2.0:1784948966.836631:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000100:00000200:2.0:1784948966.836641:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289664896, offset 0 00000100:00000001:3.0:1784948966.836643:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948966.836650:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948966.836654:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948966.836655:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784948966.836666:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784948966.836682:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784948966.836700:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438d80 00000400:00000200:2.0:1784948966.836712:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7fb9 [2] + 0 00000400:00000010:2.0:1784948966.836718:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001004a12d. 00000400:00000200:2.0:1784948966.836726:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006c7a4b68 00000400:00000200:2.0:1784948966.836736:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948966.836742:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948966.836750:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006c7a4b68 00000400:00000010:2.0:1784948966.836753:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000006c7a4b68. 00000100:00000001:2.0:1784948966.836761:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948966.836764:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784948966.836768:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000097ccad2 x1871651289664896 msgsize 224 00000100:00100000:2.0:1784948966.836774:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948966.836778:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784948966.836801:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784948966.836811:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:2.0:1784948966.836813:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000100:00000001:2.0:1784948966.836820:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948966.836825:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948966.836826:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948966.836834:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289664896 02000000:00000001:1.0:1784948966.836841:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948966.836846:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948966.836847:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948966.836854:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948966.836855:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948966.836864:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948966.836876:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289664896 00000100:00000001:2.0:1784948966.836877:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.836885:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784948966.836886:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1784948966.836892:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784948966.836898:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948966.836902:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948966.836939:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784948966.836944:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784948966.836948:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784948966.836954:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948966.836961:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305906219 00000020:00000010:1.0:1784948966.836964:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00000001:1.0:1784948966.836973:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1784948966.836977:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000082553b41. 00000020:00000010:1.0:1784948966.836978:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000100:00000001:2.0:1784948966.836998:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:1.0:1784948966.836999:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bf393d23. 00000020:00000010:1.0:1784948966.837012:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dccf8310. 00000020:00000010:1.0:1784948966.837018:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948966.837025:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305906219 00000100:00000040:1.0:1784948966.837035:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000010:2.0:1784948966.837037:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000824e9543. 00000100:00000001:1.0:1784948966.837042:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948966.837046:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948966.837047:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948966.837056:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.837060:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305906219 00000100:00000010:2.0:1784948966.837065:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000100:00000001:1.0:1784948966.837069:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.837070:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784948966.837081:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305906219 00000100:00000010:2.0:1784948966.837083:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f68fd7be. 00000100:00000001:2.0:1784948966.837086:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948966.837105:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948966.837106:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948966.837107:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.837112:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.837113:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948966.837117:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948966.837119:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948966.837120:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948966.837121:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948966.837124:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948966.837125:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948966.837129:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948966.837133:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 745 00000100:00000001:2.0:1784948966.837134:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948966.837138:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:1.0:1784948966.837139:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948966.837141:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948966.837143:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466820688 : -115544242730928 : ffff96e9c274c850) 00000100:00000001:0.0:1784948966.837144:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948966.837148:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948966.837150:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000097ccad2 x1871651289664896/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:627/0 lens 224/0 e 0 to 0 dl 1784948977 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948966.837178:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948966.837184:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00000001:1.0:1784948966.837188:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948966.837190:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000100:00100000:1.0:1784948966.837196:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000097ccad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289664896:12345-0@lo:400: 00000100:00000200:1.0:1784948966.837202:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289664896 00000020:00000001:1.0:1784948966.837207:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948966.837211:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948966.837215:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.837218:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948966.837220:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784948966.837225:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948966.837249:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948966.837254:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948966.837257:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948966.837260:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.837263:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948966.837267:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948966.837271:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948966.837277:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a20b4aa7. 02000000:00000001:1.0:1784948966.837280:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948966.837283:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.837286:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948966.837290:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289664896 00010000:00000001:1.0:1784948966.837294:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948966.837299:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000097ccad2 x1871651289664896/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:627/0 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948966.837311:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948966.837314:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948966.837319:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=138 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948966.837327:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948966.837332:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948966.837336:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000010:3.0:1784948966.837336:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007a6587e5. 02000000:00000001:1.0:1784948966.837344:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948966.837346:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948966.837349:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948966.837355:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948966.837361:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784948966.837366:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289664896, offset 224 00000400:00000200:1.0:1784948966.837376:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784948966.837388:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784948966.837401:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438d80 00000400:00000200:1.0:1784948966.837410:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x895d [1] + 224 00000400:00000200:1.0:1784948966.837418:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948966.837424:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948966.837429:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948966.837433:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948966.837445:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948966.837462:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948966.837473:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784948966.837476:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:1.0:1784948966.837482:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948966.837484:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784948966.837493:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.837500:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948966.837501:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784948966.837508:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1784948966.837509:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948966.837515:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.837519:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:2.0:1784948966.837522:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1784948966.837528:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000097ccad2 x1871651289664896/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:627/0 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784948966.837530:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948966.837537:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000075224c9. 00000400:00000200:2.0:1784948966.837550:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 02000000:00000001:3.0:1784948966.837551:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948966.837560:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000100:00100000:1.0:1784948966.837561:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000097ccad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289664896:12345-0@lo:400: Request processed in 374us (782us total) trans 0 rc 0/0 00000400:00000010:3.0:1784948966.837567:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4db7024. 00000100:00000001:2.0:1784948966.837572:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000010:3.0:1784948966.837584:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000100:00000200:2.0:1784948966.837585:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948966.837593:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 745 00000100:00000200:3.0:1784948966.837596:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289665024, portal 10 00000100:00000040:1.0:1784948966.837606:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948966.837608:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948966.837618:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948966.837625:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948966.837632:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dccf8310. 00000100:00000200:2.0:1784948966.837634:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948966.837640:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948966.837651:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bf393d23. 00000100:00000001:2.0:1784948966.837660:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1784948966.837661:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784948966.837665:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948966.837670:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948966.837671:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000010:1.0:1784948966.837677:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a20b4aa7. 02000000:00000001:2.0:1784948966.837679:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948966.837685:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:1.0:1784948966.837686:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784948966.837688:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948966.837692:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948966.837694:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948966.837701:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:2.0:1784948966.837704:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=138 v=5 (5 5 5 5) 00000400:00000010:1.0:1784948966.837712:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00001000:2.0:1784948966.837713:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=138 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948966.837723:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:1.0:1784948966.837724:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000096ea68f5. 00000100:00000001:2.0:1784948966.837726:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948966.837736:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784948966.837740:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948966.837744:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948966.837750:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549154432 : -115544160397184 : ffff96e9c75d1880) 00000100:00000040:2.0:1784948966.837754:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948966.837766:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948966.837772:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948966.837774:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948966.837781:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948966.837796:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948966.837862:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1784948966.837868:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289664896:0@lo:400: 00000100:00000040:3.0:1784948966.837880:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948966.837888:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948966.837890:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000100:00000001:2.0:1784948966.837895:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:3.0:1784948966.837900:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289665024, offset 0 00000100:00000040:2.0:1784948966.837949:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a612e3ff x1871651289664896/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948966.837960:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948966.837976:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948966.837981:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1784948966.837985:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000010:2.0:1784948966.837987:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000075318cf7. 00000400:00000200:3.0:1784948966.837997:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1784948966.837999:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948966.838005:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:3.0:1784948966.838006:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000040:2.0:1784948966.838009:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:3.0:1784948966.838018:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1784948966.838019:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948966.838028:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e79c6895. 00000400:00000200:3.0:1784948966.838029:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:2.0:1784948966.838039:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1784948966.838047:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3442:3442:245:4294967295] 192.168.202.137@tcp LPNI seq info [2226:2226:8:4294967295] 02000000:00000001:2.0:1784948966.838048:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948966.838054:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a612e3ff. 00000100:00000001:2.0:1784948966.838065:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948966.838071:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948966.838077:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948966.838086:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784948966.838089:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948966.838097:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948966.838103:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948966.838111:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:3.0:1784948966.838114:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000100:00000001:2.0:1784948966.838122:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948966.838132:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948966.838145:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948966.838150:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948966.838168:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948966.838175:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2116 usecs (3 jiffies) 00000100:00000001:3.0:1784948966.838180:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305906181 00000100:00000001:3.0:1784948966.838184:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305906219 00000100:00000001:3.0:1784948966.838187:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305906181 00000100:00000001:3.0:1784948966.838190:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305906219 00000100:00000001:3.0:1784948966.838194:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305906181 00000100:00000001:3.0:1784948966.838198:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305906219 00000100:00000001:3.0:1784948966.838201:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305906181 00000100:00000001:3.0:1784948966.838205:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305906219 00000100:00000001:3.0:1784948966.838207:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948966.838212:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948966.838216:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948966.838220:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948966.838260:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948966.838263:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948966.838269:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948966.838271:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948966.838275:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948966.838279:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948966.838334:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948966.838343:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000800:00000001:0.0:1784948966.838348:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948966.838351:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948966.838364:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948966.838365:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948966.838374:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948966.838377:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0e1f1a0 00000800:00000001:0.0:1784948966.838381:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948966.838383:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0e1f1a0. 00000100:00000001:2.0:1784948966.838390:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784948966.838398:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784948966.838405:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948966.838420:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438e00 00000100:00000001:2.0:1784948966.838421:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948966.838430:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948966.838442:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 162544 00000100:00000001:2.0:1784948966.838454:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784948966.838460:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948966.838461:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784948966.838473:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948966.838478:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948966.838485:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948966.838492:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948966.838495:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784948966.838501:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a7f2082b. 00000100:00000040:0.0:1784948966.838506:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000a7f2082b x1871651289665024 msgsize 224 00000100:00100000:0.0:1784948966.838512:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948966.838530:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784948966.838539:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948966.838543:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948966.838585:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784948966.838590:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289665024 02000000:00000001:1.0:1784948966.838594:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948966.838596:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948966.838598:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948966.838602:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948966.838605:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289665024 00000020:00000001:1.0:1784948966.838607:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948966.838609:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948966.838612:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948966.838615:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784948966.838618:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948966.838621:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948966.838626:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948966.838630:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784948966.838633:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948966.838635:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000020:00000010:1.0:1784948966.838641:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bf393d23. 00000020:00000010:1.0:1784948966.838645:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dccf8310. 00000020:00000010:1.0:1784948966.838648:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:1.0:1784948966.838656:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948966.838658:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948966.838660:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948966.838663:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948966.838669:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948966.838672:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948966.838673:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948966.838690:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948966.838700:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948966.838702:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948966.838709:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 721 00000100:00000040:1.0:1784948966.838713:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948966.838716:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554965376 : -115544154586240 : ffff96e9c7b5c380) 00000100:00000040:1.0:1784948966.838722:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a7f2082b x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:627/0 lens 224/0 e 0 to 0 dl 1784948977 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948966.838734:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948966.838737:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784948966.838740:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948966.838743:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00100000:1.0:1784948966.838751:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a7f2082b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289665024:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948966.838757:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289665024 00000020:00000001:1.0:1784948966.838759:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948966.838762:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948966.838764:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.838767:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948966.838770:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948966.838773:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948966.838776:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948966.838778:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948966.838780:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948966.838782:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.838784:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948966.838788:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948966.838790:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948966.838793:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a20b4aa7. 02000000:00000001:1.0:1784948966.838795:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948966.838798:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.838800:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948966.838802:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289665024 00010000:00000001:1.0:1784948966.838805:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948966.838809:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a7f2082b x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:627/0 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948966.838817:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948966.838820:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948966.838823:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=133 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948966.838827:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948966.838830:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948966.838833:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948966.838836:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948966.838838:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948966.838840:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948966.838843:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948966.838846:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784948966.838850:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289665024, offset 224 00000400:00000200:1.0:1784948966.838855:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948966.838864:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948966.838870:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3443:3443:245:4294967295] 192.168.202.137@tcp LPNI seq info [2227:2227:8:4294967295] 00000400:00000200:1.0:1784948966.838882:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948966.838889:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948966.838893:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 00000800:00000200:1.0:1784948966.838898:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948966.838943:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948966.838948:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948966.838958:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948966.838962:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948966.838965:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948966.838968:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948966.838970:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948966.838976:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a7f2082b x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:627/0 lens 224/224 e 0 to 0 dl 1784948977 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948966.838988:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a7f2082b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289665024:12345-192.168.202.137@tcp:400: Request processed in 252us (479us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948966.838999:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 721 00000100:00000040:1.0:1784948966.839003:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948966.839006:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948966.839009:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948966.839012:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dccf8310. 00000020:00000010:1.0:1784948966.839017:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948966.839022:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bf393d23. 00000020:00000040:1.0:1784948966.839026:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948966.839029:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948966.839149:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948966.839157:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784948966.839157:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000400:00000200:2.0:1784948966.839168:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948966.839170:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948966.839179:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948966.839180:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784948966.839186:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948966.839189:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:2.0:1784948966.839197:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000400:00000200:0.0:1784948966.839200:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948966.839205:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948966.839210:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784948966.839217:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438e00 00000400:00000200:0.0:1784948966.839226:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8969 [1] + 224 00000800:00000001:0.0:1784948966.839253:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948966.839265:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948966.839268:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948966.839274:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948966.839279:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948966.839285:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948966.839299:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948966.839320:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948966.839326:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948966.839329:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948966.839364:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948966.839368:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948966.839374:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948966.839376:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948966.839378:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000400:00000200:3.0:1784948966.839385:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000010:3.0:1784948966.839387:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000100:00000001:3.0:1784948966.839391:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948966.839397:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948966.839409:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948966.839418:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948966.839420:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948966.839422:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948966.839425:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948966.839427:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948966.839429:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948966.839431:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948966.839437:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=133 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948966.839441:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=133 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948966.839444:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948966.839446:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948966.839448:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948966.839452:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948966.839461:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948966.839465:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948966.839467:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948966.839469:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948966.839471:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948966.839475:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948966.839483:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948966.839485:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948966.839487:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948966.839490:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948966.839498:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb5afc66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289665024:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948966.839503:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948966.839506:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948966.839509:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb5afc66 x1871651289665024/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948966.839517:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948966.839519:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948966.839521:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007a6587e5. 02000000:00000001:3.0:1784948966.839527:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948966.839528:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948966.839530:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948966.839532:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948966.839534:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000031deb17e. 02000000:00000001:3.0:1784948966.839538:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948966.839539:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948966.839541:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb5afc66. 00000100:00000001:3.0:1784948966.839545:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948966.839546:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948966.839548:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948966.839552:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948966.839554:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948966.839556:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948966.839559:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948966.880024:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305906219, at 4305906220 00000100:00000010:2.0:1784948966.880038:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000040:1.0:1784948966.880054:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305906219, at 4305906220 00000100:00000040:0.0:1784948966.880060:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305906219, at 4305906220 00000100:00000040:3.0:1784948966.880064:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305906219, at 4305906220 00000100:00000010:1.0:1784948966.880069:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000824e9543. 00000100:00000010:0.0:1784948966.880075:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000100:00000010:3.0:1784948966.880081:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f68fd7be. 00000800:00000100:1.0:1784948967.028081:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000800:00000010:1.0:1784948967.028088:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009ff40a64. 00000400:00000200:1.0:1784948967.028098:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948967.028105:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948967.028112:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11240 health :900 00000400:00000200:1.0:1784948967.028119:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948967.028123:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:1.0:1784948967.028128:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000800:00000100:1.0:1784948967.028156:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000800:00000010:1.0:1784948967.028159:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003ed79484. 00000400:00000200:1.0:1784948967.028163:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948967.028168:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948967.028172:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948967.028176:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:1.0:1784948967.028181:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000200:2.0:1784948967.028205:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948967.028220:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:2.0:1784948967.028276:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948967.028280:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948967.028284:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:2.0:1784948967.028288:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000400:00000200:2.0:1784948967.028296:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948967.028304:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:2.0:1784948967.028310:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000061851253. 00000400:00000200:2.0:1784948967.028320:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:2.0:1784948967.028325:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:2.0:1784948967.028332:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:2.0:1784948967.028339:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:2.0:1784948967.028344:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948967.028346:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948967.028349:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:2.0:1784948967.028352:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:2.0:1784948967.028356:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948967.028361:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:2.0:1784948967.028365:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e7299d92. 00000400:00000200:2.0:1784948967.028373:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:2.0:1784948967.028377:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:2.0:1784948967.028382:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00010000:00000001:2.0:1784948967.668049:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948967.668059:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948967.668068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948967.668070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948967.668139:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948967.668146:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948967.668150:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948967.860031:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948967.860047:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948967.860050:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948967.860056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948967.860060:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948967.860064:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948967.860067:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784948968.052027:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000400:00000200:3.0:1784948968.052034:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000010:1.0:1784948968.052037:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000353a5c12. 00000400:00000010:3.0:1784948968.052047:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000008d3e21a4. 00000400:00000200:1.0:1784948968.052051:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784948968.052059:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003f3c01c8. 00000400:00000200:1.0:1784948968.052068:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784948968.052072:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000400:00000200:1.0:1784948968.052080:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11241 health :700 00000400:00000200:1.0:1784948968.052090:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:3.0:1784948968.052095:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005243e367. 00000400:00000200:1.0:1784948968.052096:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:3.0:1784948968.052114:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948968.052115:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:3.0:1784948968.052155:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948968.052166:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948968.052173:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3444:3444:247:4294967295] 192.168.202.137@tcp LPNI seq info [2228:2228:8:4294967295] 00000400:00000200:3.0:1784948968.052182:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948968.052191:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948968.052198:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948968.052204:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000400:00000200:2.0:1784948968.052210:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784948968.052244:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784948968.052252:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000800:00000100:1.0:1784948968.052255:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000200:3.0:1784948968.052268:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:2.0:1784948968.052271:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000010:1.0:1784948968.052272:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e5f52141. 00000800:00000200:3.0:1784948968.052278:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000001:2.0:1784948968.052280:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948968.052290:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000200:1.0:1784948968.052291:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:2.0:1784948968.052299:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:1.0:1784948968.052311:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948968.052314:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948968.052324:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948968.052328:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:2.0:1784948968.052338:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c73b3f00. 00000400:00000200:1.0:1784948968.052339:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:2.0:1784948968.052356:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000200:1.0:1784948968.052361:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:2.0:1784948968.052367:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:2.0:1784948968.052378:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000400:00000200:2.0:1784948968.052390:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:2.0:1784948968.052395:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948968.052397:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948968.052400:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:2.0:1784948968.052402:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000400:00000200:2.0:1784948968.052406:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948968.052409:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:2.0:1784948968.052413:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000070ada26c. 00000400:00000200:2.0:1784948968.052421:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:2.0:1784948968.052426:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:2.0:1784948968.052430:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000800:00000200:0.0:1784948968.052488:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948968.052489:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784948968.052498:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000800:00000001:2.0:1784948968.052508:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948968.052509:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948968.052519:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948968.052527:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784948968.052531:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948968.052534:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948968.052547:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:2.0:1784948968.052555:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948968.052576:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948968.052587:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948968.052601:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948968.052607:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3445:3445:249:4294967295] 192.168.202.137@tcp LPNI seq info [2229:2229:8:4294967295] 00000400:00000200:2.0:1784948968.052619:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948968.052625:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948968.052630:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a934b7fa. 00000800:00000200:2.0:1784948968.052637:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948968.052643:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948968.052647:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a934b7fa type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948968.052710:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948968.052766:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948968.052777:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948968.052781:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948968.052787:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948968.052788:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784948968.052798:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a934b7fa. 00000400:00000200:0.0:1784948968.052808:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948968.052809:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948968.052823:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948968.052835:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8979 00000800:00000001:0.0:1784948968.052844:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948968.052852:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948968.052860:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948968.052867:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948968.052875:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948968.052886:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000010:0.0:1784948968.052889:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000400:00000200:0.0:1784948968.052893:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948968.052896:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000008d3e21a4. 00000400:00000010:0.0:1784948968.052900:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003f3c01c8. 00000800:00000001:0.0:1784948968.052944:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948968.052949:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948968.692048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948968.692058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948968.692066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948968.692078:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948968.692163:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948968.692170:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948968.692174:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948968.883995:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948968.884010:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948968.884012:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948968.884018:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948968.884021:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948968.884025:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948968.884028:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784948969.076093:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000800:00000010:1.0:1784948969.076101:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b4c31151. 00000400:00000200:1.0:1784948969.076112:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948969.076119:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948969.076128:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11242 health :900 00000400:00000200:1.0:1784948969.076134:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948969.076139:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:1.0:1784948969.076145:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:2.0:1784948969.076252:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948969.076262:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:2.0:1784948969.076270:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948969.076273:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948969.076277:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000010:2.0:1784948969.076280:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000400:00000200:2.0:1784948969.076285:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948969.076290:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:2.0:1784948969.076295:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000859865aa. 00000400:00000200:2.0:1784948969.076302:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:2.0:1784948969.076305:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:2.0:1784948969.076310:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00010000:00000001:2.0:1784948969.716037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948969.716047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948969.716055:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948969.716057:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948969.716141:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948969.716148:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948969.716152:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948969.908017:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948969.908031:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948969.908034:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948969.908039:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948969.908043:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948969.908047:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948969.908050:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948970.100058:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784948970.100068:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000000086235. 00000400:00000010:3.0:1784948970.100081:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000041892d66. 00000400:00000010:3.0:1784948970.100088:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bb55de7f. 00000400:00000010:3.0:1784948970.100097:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000092c40cf9. 00000400:00000200:3.0:1784948970.100103:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948970.100121:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948970.100130:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948970.100136:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3446:3446:250:4294967295] 192.168.202.137@tcp LPNI seq info [2230:2230:8:4294967295] 00000400:00000200:3.0:1784948970.100144:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948970.100154:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948970.100161:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948970.100167:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948970.100175:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948970.100186:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948970.100191:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784948970.100242:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11244 health :900 00000800:00000200:2.0:1784948970.100492:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948970.100501:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:2.0:1784948970.100511:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948970.100512:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948970.100521:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:3.0:1784948970.100531:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948970.100532:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:3.0:1784948970.100543:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948970.100545:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784948970.100553:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948970.100567:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784948970.100582:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784948970.100594:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784948970.100606:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948970.100612:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3447:3447:250:4294967295] 192.168.202.137@tcp LPNI seq info [2231:2231:8:4294967295] 00000400:00000200:3.0:1784948970.100623:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784948970.100629:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948970.100632:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784948970.100638:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948970.100643:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948970.100647:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 3, nob 144 niov 1 nkiov 1 00000800:00000200:3.0:1784948970.100778:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948970.100784:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000400:00000200:3.0:1784948970.100790:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948970.100793:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948970.100803:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784948970.100806:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948970.100810:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948970.100822:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948970.100828:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948970.100832:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948970.100844:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784948970.100860:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x897d 00000800:00000001:2.0:1784948970.100866:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948970.100879:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948970.100882:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948970.100887:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784948970.100893:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bb55de7f 00000400:00000010:2.0:1784948970.100897:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bb55de7f. 00000400:00000200:2.0:1784948970.100901:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784948970.100932:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000000086235. 00000400:00000010:2.0:1784948970.100937:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000041892d66. 00000800:00000001:2.0:1784948970.100946:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948970.100950:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948970.740039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948970.740050:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948970.740058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948970.740062:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948970.740065:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948970.740069:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948970.740072:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948970.740075:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948970.740077:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948970.740080:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948970.740167:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948970.740174:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948970.740179:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948970.932042:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948970.932057:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948970.932059:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948970.932064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948970.932067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948970.932072:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948970.932075:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784948971.124042:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.27@tcp 00000800:00000010:1.0:1784948971.124053:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007a3f8609. 00000400:00000200:1.0:1784948971.124065:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948971.124078:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.27@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948971.124091:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11244 health :900 00000400:00000200:1.0:1784948971.124127:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948971.124134:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.27@tcp: 1 00000400:00000200:1.0:1784948971.124142:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.27@tcp: -114 00000400:00000200:2.0:1784948971.124255:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948971.124269:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.27@tcp(00000000898cc47a) state 0x840 00000400:00000001:2.0:1784948971.124279:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.124283:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.124287:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000010:2.0:1784948971.124292:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000400:00000200:2.0:1784948971.124298:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.124305:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.27@tcp: -114 00000400:00000010:2.0:1784948971.124311:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005cc3cd5d. 00000400:00000200:2.0:1784948971.124324:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.27@tcp:-110 00000400:00000200:2.0:1784948971.124327:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.27@tcp(00000000898cc47a) state 0x2040 rc -110 00000400:00000200:2.0:1784948971.124331:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.27@tcp 00000400:00000200:3.0:1784948971.124406:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.27@tcp NID 111.222.173.27@tcp: -110. discovery complete 00000400:00000200:3.0:1784948971.124417:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.173.27@tcp primary NID 111.222.173.27@tcp rc -110 00000100:00000010:3.0:1784948971.124436:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000dd924282. 00000100:00000001:3.0:1784948971.124481:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784948971.124484:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000dd924282 refcount 1 to 111.222.173.27@tcp 00010000:00000010:3.0:1784948971.124490:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000176cd51f. 00010000:00080000:3.0:1784948971.124545:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.173.27@tcp at tail 00010000:00000001:3.0:1784948971.124549:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948971.124554:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948971.124578:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 000000005b888794. 00000100:00000400:3.0:1784948971.124584:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.142.27@tcp: rc = -75 00000100:00000040:3.0:1784948971.124588:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.142.27@tcp cnt=32 00000400:00000200:3.0:1784948971.124591:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784948971.124595:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784948971.124598:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784948971.124600:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784948971.124603:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784948971.124610:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784948971.124616:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.124619:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.124623:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784948971.124625:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784948971.124629:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948971.124650:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784948971.124654:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.124658:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.124662:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.124666:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784948971.124669:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784948971.124676:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948971.124684:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784948971.124689:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.124691:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.124695:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.124699:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:2.0:1784948971.124703:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784948971.124703:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784948971.124708:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.124710:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784948971.124716:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:3.0:1784948971.124719:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.124724:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.124728:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000010:2.0:1784948971.124729:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000075318cf7. 00000400:00000200:3.0:1784948971.124732:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784948971.124735:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000010:2.0:1784948971.124737:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000400:00000200:3.0:1784948971.124740:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948971.124746:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:3.0:1784948971.124749:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948971.124756:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000076669228. 00000400:00000200:3.0:1784948971.124759:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.124764:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:2.0:1784948971.124765:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784948971.124768:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784948971.124771:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784948971.124775:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.124781:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:2.0:1784948971.124790:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.124797:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.124802:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.124824:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:2.0:1784948971.124831:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3448:3448:251:4294967295] 111.222.173.1@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:2.0:1784948971.124841:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:2.0:1784948971.124853:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.124862:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784948971.124862:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784948971.124865:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.124868:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000010:2.0:1784948971.124869:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000400:00000200:3.0:1784948971.124872:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.124875:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:3.0:1784948971.124878:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000800:00000200:2.0:1784948971.124879:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784948971.124883:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000800:00000200:2.0:1784948971.124884:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948971.124889:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000200:3.0:1784948971.124892:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.1:1784948971.124894:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784948971.124895:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.124899:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000800:00000200:2.1:1784948971.124900:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948971.124903:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000800:00000100:2.0:1784948971.124907:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.1@tcp 00000800:00000010:2.0:1784948971.124911:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000200:2.0:1784948971.124916:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948971.124943:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:2.0:1784948971.124944:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948971.124949:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.124950:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948971.124955:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:3.0:1784948971.124956:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784948971.124959:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948971.124964:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:3.0:1784948971.124967:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.124971:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.124974:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784948971.124978:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:2.0:1784948971.124980:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:3.0:1784948971.124982:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.124984:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 rc 0 00000400:00000200:3.0:1784948971.124988:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:3.0:1784948971.124991:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.124994:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.124995:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784948971.124999:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125002:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784948971.125005:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000010:2.0:1784948971.125007:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005cc3cd5d. 00000400:00000010:2.0:1784948971.125012:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000400:00000200:3.0:1784948971.125012:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948971.125018:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:3.0:1784948971.125021:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948971.125025:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000feb070e9. 00000400:00000200:3.0:1784948971.125025:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.125031:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:2.0:1784948971.125032:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784948971.125035:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784948971.125038:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:2.0:1784948971.125041:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784948971.125043:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125045:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.125050:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.125054:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.125058:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:2.0:1784948971.125063:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3449:3449:251:4294967295] 111.222.142.1@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:2.0:1784948971.125070:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:2.0:1784948971.125079:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.125087:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784948971.125088:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000200:3.0:1784948971.125091:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948971.125092:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000400:00000200:3.0:1784948971.125094:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948971.125098:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784948971.125098:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125111:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000200:3.0:1784948971.125114:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000800:00000200:2.1:1784948971.125115:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784948971.125119:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125121:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:2.0:1784948971.125124:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:3.0:1784948971.125125:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:3.0:1784948971.125128:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125131:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.125133:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:3.0:1784948971.125136:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125139:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784948971.125141:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000010:2.0:1784948971.125145:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ead2c724. 00000400:00000200:3.0:1784948971.125146:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948971.125150:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000400:00000200:3.0:1784948971.125152:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784948971.125171:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948971.125174:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000048122e2d. 00000400:00000200:3.0:1784948971.125174:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.125179:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:2.0:1784948971.125180:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784948971.125183:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:3.0:1784948971.125186:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:2.0:1784948971.125187:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:2.0:1784948971.125236:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948971.125236:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125242:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.125246:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.125249:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:2.0:1784948971.125254:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3450:3450:250:4294967295] 111.222.142.2@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:2.0:1784948971.125262:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:2.0:1784948971.125270:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.125278:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784948971.125279:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:3.0:1784948971.125282:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948971.125283:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076916763. 00000400:00000200:3.0:1784948971.125285:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948971.125290:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784948971.125290:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125293:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:3.0:1784948971.125297:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000800:00000200:2.1:1784948971.125297:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784948971.125302:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948971.125307:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:3.0:1784948971.125311:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125313:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.125317:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:2.0:1784948971.125320:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:3.0:1784948971.125321:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:2.0:1784948971.125323:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x261 rc 0 00000400:00000200:3.0:1784948971.125324:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:2.0:1784948971.125330:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784948971.125330:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784948971.125336:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000010:2.0:1784948971.125337:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000063f28b6f. 00000400:00000200:3.0:1784948971.125339:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125342:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784948971.125343:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000200:3.0:1784948971.125346:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125349:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000010:2.0:1784948971.125350:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c4fd57a4. 00000400:00000200:3.0:1784948971.125353:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:2.0:1784948971.125355:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784948971.125358:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125363:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:2.0:1784948971.125368:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.125373:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.125378:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.125383:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:2.0:1784948971.125389:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3451:3451:249:4294967295] 111.222.173.3@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:2.0:1784948971.125397:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:2.0:1784948971.125408:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.125415:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784948971.125417:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000800:00000010:2.0:1784948971.125421:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ca1124f5. 00000400:00000200:3.0:1784948971.125421:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125424:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948971.125429:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784948971.125429:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125433:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000800:00000200:2.0:1784948971.125434:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948971.125437:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:3.0:1784948971.125443:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784948971.125443:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784948971.125447:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948971.125450:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784948971.125454:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000100:2.0:1784948971.125455:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.3@tcp 00000800:00000010:2.0:1784948971.125458:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ca1124f5. 00000400:00000200:3.0:1784948971.125458:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.125463:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948971.125463:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125484:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:2.0:1784948971.125486:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948971.125487:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:2.0:1784948971.125491:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948971.125494:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125496:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:3.0:1784948971.125501:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:3.0:1784948971.125504:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948971.125508:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:3.0:1784948971.125511:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.125516:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125520:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:3.0:1784948971.125523:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:2.0:1784948971.125529:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784948971.125529:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125534:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 rc 0 00000400:00000200:3.0:1784948971.125536:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:3.0:1784948971.125539:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125542:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.125544:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784948971.125548:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125551:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784948971.125554:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000010:2.0:1784948971.125555:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000072ce2c8. 00000400:00000200:3.0:1784948971.125560:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948971.125561:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000200:3.0:1784948971.125566:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:3.0:1784948971.125569:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948971.125573:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004656e9af. 00000400:00000200:3.0:1784948971.125573:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.125582:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784948971.125582:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125586:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:3.0:1784948971.125590:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:2.0:1784948971.125592:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:2.0:1784948971.125597:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948971.125597:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125604:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.125610:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.125616:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:2.0:1784948971.125622:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3452:3452:249:4294967295] 111.222.173.4@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:2.0:1784948971.125628:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:2.0:1784948971.125637:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.125644:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784948971.125645:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:3.0:1784948971.125648:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948971.125649:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ca1124f5. 00000400:00000200:3.0:1784948971.125651:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948971.125657:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784948971.125657:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.125661:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000800:00000200:2.1:1784948971.125664:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784948971.125665:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:3.0:1784948971.125670:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125671:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:2.0:1784948971.125675:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784948971.125677:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:3.0:1784948971.125681:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125684:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.125685:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000200:3.0:1784948971.125689:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000001:2.0:1784948971.125690:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.125693:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784948971.125693:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:2.0:1784948971.125696:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000200:3.0:1784948971.125697:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000010:2.0:1784948971.125699:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000400:00000200:3.0:1784948971.125702:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125705:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948971.125708:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784948971.125711:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948971.125715:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:2.0:1784948971.125719:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000075318cf7. 00000400:00000200:3.0:1784948971.125724:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.125725:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-5 00000400:00000200:3.0:1784948971.125729:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:2.0:1784948971.125730:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -5 00000400:00000200:3.0:1784948971.125733:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:2.0:1784948971.125735:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000400:00000200:3.0:1784948971.125736:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:2.0:1784948971.125740:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000200:3.0:1784948971.125741:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784948971.125746:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000075318cf7. 00000400:00000200:3.0:1784948971.125748:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000010:2.0:1784948971.125750:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000400:00000200:3.0:1784948971.125751:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125755:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.125759:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000010:2.0:1784948971.125763:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004bacba68. 00000400:00000200:3.0:1784948971.125763:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:3.0:1784948971.125766:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:2.0:1784948971.125769:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:3.0:1784948971.125772:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125776:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:2.0:1784948971.125780:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.125784:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.125788:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.125791:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:2.0:1784948971.125797:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3453:3453:248:4294967295] 111.222.142.4@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:2.0:1784948971.125805:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:2.0:1784948971.125830:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.125838:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:2.0:1784948971.125842:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a21d8ef3. 00000800:00000200:2.0:1784948971.125848:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784948971.125854:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:2.0:1784948971.125879:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:2.0:1784948971.125883:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x261 rc 0 00000400:00000200:2.0:1784948971.125943:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000200:3.0:1784948971.125945:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:3.0:1784948971.125949:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.125952:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784948971.125956:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000200:3.0:1784948971.125960:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000010:2.0:1784948971.125961:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000400:00000200:3.0:1784948971.125965:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000200:3.0:1784948971.125968:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000010:2.0:1784948971.125969:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f2e2ddd3. 00000400:00000200:3.0:1784948971.125974:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.125975:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:2.0:1784948971.125985:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:2.0:1784948971.125991:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.125996:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126002:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126007:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:2.0:1784948971.126014:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3454:3454:247:4294967295] 111.222.173.5@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:2.0:1784948971.126023:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:2.0:1784948971.126035:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.126044:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000400:00000200:3.0:1784948971.126047:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784948971.126050:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948971.126051:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000400:00000200:3.0:1784948971.126054:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.126058:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000800:00000200:2.0:1784948971.126059:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:3.0:1784948971.126062:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000800:00000200:2.0:1784948971.126063:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948971.126066:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:3.0:1784948971.126072:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784948971.126072:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.1:1784948971.126077:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784948971.126080:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:3.0:1784948971.126084:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000100:2.0:1784948971.126085:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.5@tcp 00000800:00000010:2.0:1784948971.126088:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000400:00000200:3.0:1784948971.126088:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.126094:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:2.0:1784948971.126095:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948971.126099:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000200:2.0:1784948971.126103:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784948971.126103:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:2.0:1784948971.126109:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948971.126110:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.126114:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:3.0:1784948971.126117:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:3.0:1784948971.126120:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784948971.126124:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:3.0:1784948971.126127:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784948971.126131:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.126134:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:3.0:1784948971.126137:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:2.0:1784948971.126138:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:2.0:1784948971.126142:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 rc 0 00000400:00000200:2.0:1784948971.126147:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:2.0:1784948971.126152:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000059e5b358. 00000400:00000010:2.0:1784948971.126157:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000492bb2ef. 00000400:00000010:2.0:1784948971.126163:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000071cfa864. 00000400:00000200:2.0:1784948971.126168:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:2.0:1784948971.126175:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:2.0:1784948971.126180:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.126184:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126188:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126215:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:2.0:1784948971.126221:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3455:3455:247:4294967295] 111.222.142.5@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:2.0:1784948971.126229:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:3.0:1784948971.126236:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.126239:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.126246:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000400:00000200:3.0:1784948971.126248:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784948971.126251:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784948971.126252:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000400:00000200:3.0:1784948971.126254:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784948971.126259:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:3.0:1784948971.126259:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.126263:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000800:00000200:2.1:1784948971.126265:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:3.0:1784948971.126266:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000200:3.0:1784948971.126271:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784948971.126272:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:2.0:1784948971.126275:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:3.0:1784948971.126278:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000200:3.0:1784948971.126281:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784948971.126284:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784948971.126285:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000200:3.0:1784948971.126288:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784948971.126291:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000400:00000010:2.0:1784948971.126295:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000011003cc7. 00000100:00000010:3.0:1784948971.126296:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 000000005b888794. 00000400:00000010:2.0:1784948971.126299:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000400:00000010:2.0:1784948971.126306:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f3516823. 00000020:00000001:3.0:1784948971.126306:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00010000:00000001:3.0:1784948971.126310:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000400:00000200:2.0:1784948971.126311:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000100:00000001:3.0:1784948971.126316:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:2.0:1784948971.126318:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:2.0:1784948971.126322:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.126326:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126330:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126334:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:2.0:1784948971.126338:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3456:3456:246:4294967295] 111.222.142.6@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784948971.126345:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:2.0:1784948971.126354:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.126361:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:2.0:1784948971.126365:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000094ed938f. 00000400:00000010:3.0:1784948971.126365:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 0000000041892d66. 00000400:00000200:3.0:1784948971.126371:0:343529:0:(peer.c:266:lnet_peer_alloc()) 0000000041892d66 nid 111.222.142.27@tcp 00000800:00000200:2.0:1784948971.126372:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000010:3.0:1784948971.126378:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000393cfc50. 00000800:00000200:2.1:1784948971.126378:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000200:3.0:1784948971.126382:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 00000000393cfc50 net tcp 00000400:00000010:3.0:1784948971.126387:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000003f3c01c8. 00000400:00000200:3.0:1784948971.126392:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 000000003f3c01c8 nid 111.222.142.27@tcp 00000400:00000200:2.0:1784948971.126402:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:3.0:1784948971.126403:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.142.27@tcp NID 111.222.142.27@tcp flags 0x0 00000400:00000200:2.0:1784948971.126406:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x261 rc 0 00000400:00000200:3.0:1784948971.126410:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.142.27@tcp: 0 00000400:00000200:2.0:1784948971.126412:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000200:3.0:1784948971.126417:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.27@tcp: 0 00000400:00000200:3.0:1784948971.126420:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784948971.126426:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fff5e130. 00000400:00000010:2.0:1784948971.126432:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000010:2.0:1784948971.126440:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d9c0bef1. 00000400:00000200:2.0:1784948971.126447:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:2.0:1784948971.126456:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:2.0:1784948971.126459:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.126464:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126469:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126475:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:2.0:1784948971.126482:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3457:3457:245:4294967295] 111.222.173.7@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:2.0:1784948971.126489:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:2.0:1784948971.126501:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.126510:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:2.0:1784948971.126516:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c53910d9. 00000800:00000200:2.0:1784948971.126522:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.0:1784948971.126527:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948971.126536:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784948971.126540:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948971.126546:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.7@tcp 00000800:00000010:2.0:1784948971.126549:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c53910d9. 00000400:00000200:2.0:1784948971.126556:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948971.126563:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948971.126569:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948971.126575:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:2.0:1784948971.126583:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:2.0:1784948971.126590:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:2.0:1784948971.126592:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 rc 0 00000400:00000200:2.0:1784948971.126596:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:2.0:1784948971.126600:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000053ab16eb. 00000400:00000010:2.0:1784948971.126604:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000010:2.0:1784948971.126607:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007a213df0. 00000400:00000200:2.0:1784948971.126610:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:2.0:1784948971.126616:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:2.0:1784948971.126619:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.126622:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126625:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126628:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:2.0:1784948971.126631:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3458:3458:245:4294967295] 111.222.173.8@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:2.0:1784948971.126635:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:2.0:1784948971.126641:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.126645:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784948971.126648:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c53910d9. 00000800:00000200:2.0:1784948971.126652:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784948971.126656:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:2.0:1784948971.126660:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:2.0:1784948971.126662:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:2.0:1784948971.126665:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:2.0:1784948971.126671:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f1904ca. 00000400:00000010:2.0:1784948971.126673:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000400:00000010:2.0:1784948971.126677:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000056f20560. 00000400:00000200:2.0:1784948971.126680:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:2.0:1784948971.126683:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:2.0:1784948971.126686:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.126689:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126691:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126694:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:2.0:1784948971.126696:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3459:3459:244:4294967295] 111.222.142.8@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784948971.126701:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:2.0:1784948971.126707:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.126711:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784948971.126714:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f495c80e. 00000800:00000200:2.0:1784948971.126719:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784948971.126725:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000400:00000200:2.0:1784948971.126745:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:2.0:1784948971.126748:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x261 rc 0 00000400:00000200:2.0:1784948971.126753:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:2.0:1784948971.126759:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dcf1bf81. 00000400:00000010:2.0:1784948971.126764:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000400:00000010:2.0:1784948971.126770:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000619517d8. 00000400:00000200:2.0:1784948971.126776:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:2.0:1784948971.126783:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:2.0:1784948971.126787:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.126792:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126796:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126800:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:2.0:1784948971.126805:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3460:3460:243:4294967295] 111.222.173.9@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:2.0:1784948971.126811:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:2.0:1784948971.126820:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.126828:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:2.0:1784948971.126834:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000005678171. 00000800:00000200:2.0:1784948971.126841:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.0:1784948971.126845:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948971.126852:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784948971.126855:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948971.126861:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.9@tcp 00000800:00000010:2.0:1784948971.126865:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000005678171. 00000400:00000200:2.0:1784948971.126870:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948971.126875:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948971.126880:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948971.126884:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:2.0:1784948971.126889:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:2.0:1784948971.126898:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:2.0:1784948971.126901:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 rc 0 00000400:00000200:2.0:1784948971.126940:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:2.0:1784948971.126948:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004520da55. 00000400:00000010:2.0:1784948971.126953:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000400:00000010:2.0:1784948971.126959:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004c587164. 00000400:00000200:2.0:1784948971.126964:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:2.0:1784948971.126972:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:2.0:1784948971.126976:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.126981:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.126985:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.126989:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:2.0:1784948971.126994:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3461:3461:243:4294967295] 111.222.142.9@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:2.0:1784948971.127002:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:2.0:1784948971.127011:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127019:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:2.0:1784948971.127024:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000005678171. 00000800:00000200:2.0:1784948971.127032:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784948971.127039:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:2.0:1784948971.127047:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:2.0:1784948971.127051:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:2.0:1784948971.127059:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:2.0:1784948971.127064:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.127067:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.127069:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:2.0:1784948971.127073:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:2.0:1784948971.127079:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.127086:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:2.0:1784948971.127091:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000063f28b6f. 00000400:00000200:2.0:1784948971.127100:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-5 00000400:00000200:2.0:1784948971.127105:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -5 00000400:00000200:2.0:1784948971.127109:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:2.0:1784948971.127114:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:2.0:1784948971.127138:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000b241612. 00000400:00000010:2.0:1784948971.127143:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000010:2.0:1784948971.127151:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004cc664c0. 00000400:00000200:2.0:1784948971.127158:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:2.0:1784948971.127166:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:2.0:1784948971.127171:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.127176:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.127180:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.127184:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:2.0:1784948971.127190:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3462:3462:242:4294967295] 111.222.173.10@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:2.0:1784948971.127245:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:2.0:1784948971.127255:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127264:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784948971.127269:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007a3f8609. 00000800:00000200:2.0:1784948971.127274:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784948971.127279:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:2.0:1784948971.127284:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:2.0:1784948971.127286:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:2.0:1784948971.127290:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:2.0:1784948971.127294:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ee879bbd. 00000400:00000010:2.0:1784948971.127297:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000400:00000010:2.0:1784948971.127302:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000049855afc. 00000400:00000200:2.0:1784948971.127305:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:2.0:1784948971.127310:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:2.0:1784948971.127313:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.127316:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.127319:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.127322:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:2.0:1784948971.127325:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3463:3463:241:4294967295] 111.222.142.10@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:2.0:1784948971.127329:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:2.0:1784948971.127335:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127338:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:2.0:1784948971.127343:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:2.0:1784948971.127346:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784948971.127350:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000400:00000200:2.0:1784948971.127368:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:2.0:1784948971.127370:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x261 rc 0 00000400:00000200:2.0:1784948971.127374:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:2.0:1784948971.127377:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006841dfc4. 00000400:00000010:2.0:1784948971.127379:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000400:00000010:2.0:1784948971.127383:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a7d88830. 00000400:00000200:2.0:1784948971.127386:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:2.0:1784948971.127390:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:2.0:1784948971.127393:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.127396:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.127398:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.127401:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:2.0:1784948971.127405:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3464:3464:240:4294967295] 111.222.173.11@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:2.0:1784948971.127411:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:2.0:1784948971.127419:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127425:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784948971.127428:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000800:00000200:2.0:1784948971.127434:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.0:1784948971.127436:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948971.127461:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784948971.127463:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948971.127468:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784948971.127471:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000200:2.0:1784948971.127474:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948971.127481:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948971.127487:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948971.127491:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:2.0:1784948971.127494:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:2.0:1784948971.127504:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:2.0:1784948971.127508:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 rc 0 00000400:00000200:2.0:1784948971.127512:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:2.0:1784948971.127519:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000cf77ef26. 00000400:00000010:2.0:1784948971.127522:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000400:00000010:2.0:1784948971.127527:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000060798c14. 00000400:00000200:2.0:1784948971.127531:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:2.0:1784948971.127537:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:2.0:1784948971.127540:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.127543:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.127545:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.127548:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:2.0:1784948971.127551:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3465:3465:240:4294967295] 111.222.173.12@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1784948971.127557:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:2.0:1784948971.127567:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127573:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784948971.127578:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000800:00000200:2.0:1784948971.127583:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784948971.127588:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:2.0:1784948971.127611:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:2.0:1784948971.127614:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:2.0:1784948971.127617:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:2.0:1784948971.127621:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004f865a97. 00000400:00000010:2.0:1784948971.127624:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000400:00000010:2.0:1784948971.127628:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000062d84ae9. 00000400:00000200:2.0:1784948971.127632:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:2.0:1784948971.127637:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:2.0:1784948971.127640:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.127643:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.127646:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.127649:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:2.0:1784948971.127652:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3466:3466:239:4294967295] 111.222.142.12@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:2.0:1784948971.127657:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:2.0:1784948971.127663:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127667:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784948971.127671:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b9b32818. 00000800:00000200:2.0:1784948971.127675:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784948971.127679:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:2.0:1784948971.127700:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:2.0:1784948971.127703:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:2.0:1784948971.127707:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:2.0:1784948971.127711:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000063f28b6f. 00000400:00000010:2.0:1784948971.127713:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000400:00000010:2.0:1784948971.127717:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000473e84ce. 00000400:00000200:2.0:1784948971.127722:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:2.0:1784948971.127729:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:2.0:1784948971.127736:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.127744:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.127749:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.127753:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:2.0:1784948971.127759:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3467:3467:238:4294967295] 111.222.173.13@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:2.0:1784948971.127769:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:2.0:1784948971.127781:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127789:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:2.0:1784948971.127797:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cee99234. 00000800:00000200:2.0:1784948971.127822:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.0:1784948971.127826:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.13 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948971.127833:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784948971.127838:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.13 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948971.127846:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.13@tcp 00000800:00000010:2.0:1784948971.127850:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cee99234. 00000400:00000200:2.0:1784948971.127856:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948971.127862:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948971.127867:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948971.127871:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:2.0:1784948971.127876:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:2.0:1784948971.127884:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:2.0:1784948971.127886:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 rc 0 00000400:00000200:2.0:1784948971.127890:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:2.0:1784948971.127896:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006200396a. 00000400:00000010:2.0:1784948971.127899:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000010:2.0:1784948971.127903:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000775d3d92. 00000400:00000200:2.0:1784948971.127908:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:2.0:1784948971.127916:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:2.0:1784948971.127957:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.127961:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.127965:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.127969:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:2.0:1784948971.127974:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3468:3468:238:4294967295] 111.222.142.13@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:2.0:1784948971.127981:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:2.0:1784948971.127991:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.127999:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:2.0:1784948971.128006:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cee99234. 00000800:00000200:2.0:1784948971.128013:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784948971.128020:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:2.0:1784948971.128026:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:2.0:1784948971.128030:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:2.0:1784948971.128036:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:2.0:1784948971.128042:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000d531934e. 00000400:00000010:2.0:1784948971.128047:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000400:00000010:2.0:1784948971.128053:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e4a8defd. 00000400:00000200:2.0:1784948971.128058:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:2.0:1784948971.128067:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:2.0:1784948971.128071:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.128076:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.128079:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.128081:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:2.0:1784948971.128084:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3469:3469:237:4294967295] 111.222.173.14@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:2.0:1784948971.128089:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:2.0:1784948971.128095:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.128100:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784948971.128104:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cf155122. 00000800:00000200:2.0:1784948971.128130:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784948971.128134:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:2.0:1784948971.128137:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:2.0:1784948971.128139:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:2.0:1784948971.128142:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:2.0:1784948971.128145:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000307ddfde. 00000400:00000010:2.0:1784948971.128147:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000010:2.0:1784948971.128150:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000074e9a075. 00000400:00000200:2.0:1784948971.128153:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:2.0:1784948971.128157:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:2.0:1784948971.128160:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.128162:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.128164:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.128166:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:2.0:1784948971.128168:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3470:3470:236:4294967295] 111.222.142.14@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:2.0:1784948971.128172:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:2.0:1784948971.128215:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.128221:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:2.0:1784948971.128225:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076202100. 00000800:00000200:2.0:1784948971.128230:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.1:1784948971.128233:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:2.0:1784948971.128249:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:2.0:1784948971.128251:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:2.0:1784948971.128255:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 00000400:00000001:2.0:1784948971.128258:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.128259:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.128261:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:2.0:1784948971.128263:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000400:00000200:2.0:1784948971.128266:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.128269:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:2.0:1784948971.128272:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b1cbf66f. 00000400:00000200:2.0:1784948971.128278:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-5 00000400:00000200:2.0:1784948971.128281:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -5 00000400:00000200:2.0:1784948971.128285:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:2.0:1784948971.128290:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:2.0:1784948971.128294:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000010:2.0:1784948971.128297:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000400:00000010:2.0:1784948971.128301:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a816c742. 00000400:00000200:2.0:1784948971.128305:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:2.0:1784948971.128311:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:2.0:1784948971.128316:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.128320:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.128323:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.128325:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:2.0:1784948971.128329:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3471:3471:235:4294967295] 111.222.173.15@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:2.0:1784948971.128334:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:2.0:1784948971.128346:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.128352:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784948971.128357:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000beaddf6f. 00000800:00000200:2.0:1784948971.128362:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.0:1784948971.128365:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.15 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784948971.128371:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784948971.128373:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.15 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784948971.128378:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784948971.128380:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000beaddf6f. 00000400:00000200:2.0:1784948971.128386:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784948971.128390:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948971.128393:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948971.128396:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:2.0:1784948971.128399:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:2.0:1784948971.128405:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:2.0:1784948971.128407:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 rc 0 00000400:00000200:2.0:1784948971.128411:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:2.0:1784948971.128421:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000859865aa. 00000400:00000010:2.0:1784948971.128424:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000010:2.0:1784948971.128427:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006aad92e0. 00000400:00000200:2.0:1784948971.128435:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:2.0:1784948971.128440:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:2.0:1784948971.128443:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784948971.128446:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.128448:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784948971.128451:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:2.0:1784948971.128453:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3472:3472:235:4294967295] 111.222.142.15@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:2.0:1784948971.128458:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:2.0:1784948971.128464:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.128468:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:2.0:1784948971.128470:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000beaddf6f. 00000800:00000200:2.0:1784948971.128474:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784948971.128477:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:2.0:1784948971.128480:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:2.0:1784948971.128482:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:2.0:1784948971.128486:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.27@tcp(0000000041892d66) state 0x40 00000400:00000010:2.0:1784948971.128491:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef171f31. 00000400:00000010:2.0:1784948971.128495:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000400:00000010:2.0:1784948971.128501:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004f0f2833. 00000400:00000200:2.0:1784948971.128507:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.27@tcp 00000400:00000200:2.0:1784948971.128514:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.142.27@tcp local destination 00000400:00000200:2.0:1784948971.128520:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784948971.128525:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.142.27@tcp 00000400:00000200:2.0:1784948971.128531:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.142.27@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:2.0:1784948971.128537:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3473:3473:234:4294967295] 111.222.142.27@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1784948971.128544:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.27@tcp 00000400:00000200:2.0:1784948971.128553:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.27@tcp(111.222.142.27@tcp:111.222.142.27@tcp) : GET try# 0 00000800:00000200:2.0:1784948971.128561:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.27@tcp 00000800:00000010:2.0:1784948971.128566:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cb55bcd8. 00000800:00000010:2.0:1784948971.128577:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000ae75204b. 00000800:00000010:2.0:1784948971.128584:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 0000000037ca0070. 00000800:00000200:2.0:1784948971.128592:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ae75204b] -> 12345-111.222.142.27@tcp (2) 00000800:00000200:2.1:1784948971.128599:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ae75204b] -> 12345-111.222.142.27@tcp (2) 00000400:00000200:2.0:1784948971.128619:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.27@tcp 00000400:00000200:2.0:1784948971.128623:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.27@tcp(0000000041892d66) state 0x240 rc 0 00000400:00000200:2.0:1784948971.128629:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000001:2.0:1784948971.128633:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.128635:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.128638:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:2.0:1784948971.128640:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000400:00000200:2.0:1784948971.128645:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.128649:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:2.0:1784948971.128654:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fff5e130. 00000400:00000200:2.0:1784948971.128660:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-5 00000400:00000200:2.0:1784948971.128664:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -5 00000400:00000200:2.0:1784948971.128669:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000400:00000200:2.0:1784948971.128673:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000001:2.0:1784948971.128677:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.128679:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.128681:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:2.0:1784948971.128684:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000400:00000200:2.0:1784948971.128687:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.128692:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:2.0:1784948971.128695:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dcf1bf81. 00000400:00000200:2.0:1784948971.128701:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-5 00000400:00000200:2.0:1784948971.128705:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -5 00000400:00000200:2.0:1784948971.128709:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000400:00000200:2.0:1784948971.128714:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:2.0:1784948971.128718:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.128720:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.128722:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:2.0:1784948971.128724:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000400:00000200:2.0:1784948971.128728:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.128732:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:2.0:1784948971.128735:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006841dfc4. 00000400:00000200:2.0:1784948971.128741:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-5 00000400:00000200:2.0:1784948971.128745:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -5 00000400:00000200:2.0:1784948971.128751:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000400:00000200:2.0:1784948971.128756:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:2.0:1784948971.128760:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.128763:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.128765:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:2.0:1784948971.128768:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000400:00000200:2.0:1784948971.128771:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.128775:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:2.0:1784948971.128779:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000063f28b6f. 00000400:00000200:2.0:1784948971.128783:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-5 00000400:00000200:2.0:1784948971.128787:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -5 00000400:00000200:2.0:1784948971.128791:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000400:00000200:2.0:1784948971.128796:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:2.0:1784948971.128799:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948971.128802:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948971.128803:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:2.0:1784948971.128805:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000400:00000200:2.0:1784948971.128808:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948971.128811:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:2.0:1784948971.128813:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b1cbf66f. 00000400:00000200:2.0:1784948971.128816:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-5 00000400:00000200:2.0:1784948971.128818:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -5 00000400:00000200:2.0:1784948971.128821:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00010000:00000001:2.0:1784948971.764037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948971.764047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948971.764055:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948971.764058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948971.764230:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948971.764236:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948971.764241:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948971.956048:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948971.956051:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784948971.956063:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784948971.956067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948971.956072:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784948971.956074:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb4f6eba. 00000100:00000001:2.0:1784948971.956079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784948971.956088:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948971.956089:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948971.956096:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948971.956099:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784948971.956102:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948971.956106:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948971.956115:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948971.956126:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000001:1.0:1784948971.956134:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948971.956147:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948971.956162:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948971.956166:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948971.956174:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb4f6eba] to pc [ptlrpcd_00_01+1] req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948971.956262:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948971.956268:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948971.956274:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948971.956279:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948971.956284:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948971.956290:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb5afc66. 00000020:00000040:1.0:1784948971.956294:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948971.956297:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948971.956299:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948971.956302:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.956304:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1784948971.956308:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:0.0:1784948971.956312:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948971.956315:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.956318:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948971.956325:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784948971.956326:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784948971.956330:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948971.956334:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948971.956380:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948971.956382:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948971.956384:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.956387:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948971.956396:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784948971.956397:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb5afc66] to pc [ptlrpcd_00_02+2] req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948971.956404:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948971.956416:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948971.956427:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289665152:0@lo:400: 00000100:00000001:0.0:1784948971.956434:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948971.956441:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948971.956444:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948971.956447:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948971.956451:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948971.956456:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 00000100:00000001:3.0:1784948971.956463:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948971.956464:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948971.956464:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11248) 00000100:00000001:3.0:1784948971.956473:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948971.956479:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:0.0:1784948971.956482:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4e2c75c. 00000100:00000040:3.0:1784948971.956492:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948971.956495:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000200:0.0:1784948971.956513:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289665152, portal 25 00000100:00000001:1.0:1784948971.956513:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948971.956515:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948971.956523:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.956524:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948971.956524:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.956531:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549148160 : -115544160403456 : ffff96e9c75d0000) 02000000:00000001:3.0:1784948971.956536:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948971.956541:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784948971.956545:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948971.956548:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948971.956563:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb5afc66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289665280:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948971.956577:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948971.956584:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948971.956590:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.956592:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784948971.956595:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784948971.956604:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000010:3.0:1784948971.956605:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 02000000:00000001:3.0:1784948971.956611:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948971.956612:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000100:00000200:0.0:1784948971.956618:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289665152, offset 0 00000400:00000010:3.0:1784948971.956621:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4db7024. 00000400:00000200:0.0:1784948971.956630:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784948971.956650:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000400:00000200:0.0:1784948971.956651:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:3.0:1784948971.956666:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289665280, portal 10 00000100:00000001:3.0:1784948971.956674:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1784948971.956682:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438e80 00000100:00000001:3.0:1784948971.956683:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549154432 : -115544160397184 : ffff96e9c75d1880) 00000400:00000200:0.0:1784948971.956699:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7fdd [2] + 0 00000100:00000040:3.0:1784948971.956708:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948971.956709:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000968c2fa4. 00000400:00000200:0.0:1784948971.956720:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000feb5569e 00000400:00000200:0.0:1784948971.956734:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948971.956735:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784948971.956751:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:3.0:1784948971.956753:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948971.956762:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000feb5569e 00000400:00000010:3.0:1784948971.956764:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000400:00000010:0.0:1784948971.956768:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000feb5569e. 00000100:00000001:0.0:1784948971.956777:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948971.956779:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:3.0:1784948971.956782:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289665280, offset 0 00000100:00000040:0.0:1784948971.956785:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000008db39772 x1871651289665152 msgsize 224 00000100:00100000:0.0:1784948971.956795:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948971.956802:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948971.956803:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948971.956823:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784948971.956826:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948971.956833:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948971.956840:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948971.956845:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948971.956851:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948971.956858:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3474:3474:233:4294967295] 192.168.202.137@tcp LPNI seq info [2232:2232:8:4294967295] 00000400:00000200:3.0:1784948971.956879:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784948971.956885:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:0.0:1784948971.956893:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000800:00000200:3.0:1784948971.956894:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00100000:2.0:1784948971.956895:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289665152 02000000:00000001:2.0:1784948971.956949:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000010:0.0:1784948971.956952:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000100:00000001:2.0:1784948971.956955:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948971.956962:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784948971.956963:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948971.956968:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000200:0.0:1784948971.956973:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948971.956975:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948971.956988:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:2.0:1784948971.956991:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289665152 00000020:00000001:2.0:1784948971.957005:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948971.957010:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1784948971.957012:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000200:3.0:1784948971.957015:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:0.0:1784948971.957021:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948971.957022:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:3.0:1784948971.957025:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000020:00000040:2.0:1784948971.957033:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784948971.957049:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948971.957053:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948971.957057:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948971.957059:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948971.957060:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948971.957069:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948971.957074:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948971.957074:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2118 usecs (3 jiffies) 00000020:00000010:2.0:1784948971.957078:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:3.0:1784948971.957087:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305911300 00000100:00000001:0.0:1784948971.957090:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305911339 00000100:00000001:2.0:1784948971.957091:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948971.957103:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000010:3.0:1784948971.957105:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003e9ea21c. 00000020:00000010:2.0:1784948971.957118:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e5260462. 00000100:00000001:3.0:1784948971.957122:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000010:2.0:1784948971.957130:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000091201221. 00000100:00000001:3.0:1784948971.957139:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305911300 00000020:00000010:2.0:1784948971.957140:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000010:3.0:1784948971.957153:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000040:2.0:1784948971.957161:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948971.957162:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784948971.957173:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948971.957179:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305911300 00000100:00000001:2.0:1784948971.957180:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948971.957217:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948971.957218:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002b896c5a. 00000100:00000001:3.0:1784948971.957231:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948971.957238:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305911300 00000100:00000001:2.0:1784948971.957240:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948971.957246:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000100:00000001:3.0:1784948971.957255:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948971.957262:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948971.957266:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948971.957269:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948971.957270:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305911300 00000100:00000001:3.0:1784948971.957271:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.957284:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948971.957288:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305911339 00000100:00000001:3.0:1784948971.957295:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948971.957300:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948971.957302:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305911300 00000100:00000001:2.0:1784948971.957304:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948971.957306:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948971.957315:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948971.957316:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948971.957318:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305911339 00000100:00000001:3.0:1784948971.957324:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.957329:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305911300 00000100:00100000:2.0:1784948971.957333:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 746 00000100:00000001:3.0:1784948971.957335:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.957339:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305911339 00000100:00000040:2.0:1784948971.957342:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948971.957348:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466830928 : -115544242720688 : ffff96e9c274f050) 00000100:00000001:0.0:1784948971.957354:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305911300 00000100:00000040:2.0:1784948971.957359:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000008db39772 x1871651289665152/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:632/0 lens 224/0 e 0 to 0 dl 1784948982 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948971.957369:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1784948971.957369:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784948971.957370:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948971.957380:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948971.957382:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784948971.957384:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000001:0.0:1784948971.957391:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948971.957392:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000800:00000001:3.0:1784948971.957393:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948971.957399:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948971.957403:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948971.957405:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:3.0:1784948971.957406:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784948971.957417:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:0.0:1784948971.957421:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784948971.957421:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948971.957422:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948971.957429:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1784948971.957434:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000008db39772 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289665152:12345-0@lo:400: 00000100:00000001:0.0:1784948971.957442:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948971.957445:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000100:00000001:0.0:1784948971.957449:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784948971.957451:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784948971.957453:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289665152 00000400:00000010:1.0:1784948971.957455:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:0.0:1784948971.957457:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948971.957465:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948971.957470:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948971.957471:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1784948971.957473:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784948971.957482:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948971.957482:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6438f00 00000100:00000200:1.0:1784948971.957489:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948971.957495:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948971.957502:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:3.0:1784948971.957511:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 162768 00000020:00000001:2.0:1784948971.957515:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948971.957522:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948971.957532:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784948971.957533:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784948971.957537:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784948971.957541:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:3.0:1784948971.957547:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948971.957549:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948971.957550:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948971.957561:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784948971.957562:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948971.957569:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784948971.957577:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784948971.957580:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948971.957583:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948971.957585:0:337112:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:2.0:1784948971.957589:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c38711c7. 02000000:00000001:2.0:1784948971.957596:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.957597:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784948971.957600:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948971.957603:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:2.0:1784948971.957606:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784948971.957613:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ec3ca5ba. 00010000:00000040:2.0:1784948971.957614:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289665152 00010000:00000001:2.0:1784948971.957625:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:3.0:1784948971.957626:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000ec3ca5ba x1871651289665280 msgsize 224 00010000:00000200:2.0:1784948971.957636:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000008db39772 x1871651289665152/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:632/0 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948971.957658:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00100000:3.0:1784948971.957660:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:2.0:1784948971.957663:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948971.957671:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=143 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948971.957677:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948971.957681:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784948971.957684:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784948971.957687:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784948971.957690:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.957693:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784948971.957695:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948971.957705:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948971.957710:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000800:00000001:3.0:1784948971.957716:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:2.0:1784948971.957717:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289665152, offset 224 00000400:00000200:2.0:1784948971.957731:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:3.0:1784948971.957731:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948971.957752:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948971.957757:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784948971.957767:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438e80 00000400:00000200:2.0:1784948971.957775:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x89e9 [1] + 224 00000400:00000200:2.0:1784948971.957783:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948971.957788:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784948971.957793:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948971.957798:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948971.957811:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948971.957820:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289665280 02000000:00000001:0.0:1784948971.957827:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948971.957830:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948971.957834:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948971.957839:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1784948971.957840:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948971.957848:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289665280 00000020:00000001:0.0:1784948971.957852:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1784948971.957853:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000020:00000040:0.0:1784948971.957856:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:2.0:1784948971.957860:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000020:00000001:0.0:1784948971.957863:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784948971.957870:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784948971.957871:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948971.957876:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:0.0:1784948971.957877:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784948971.957884:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784948971.957886:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.957891:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784948971.957896:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000010:0.0:1784948971.957901:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784948971.957943:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784948971.957944:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784948971.957946:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784948971.957950:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948971.957954:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784948971.957955:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784948971.957957:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:3.0:1784948971.957970:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784948971.957971:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008db39772 x1871651289665152/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:632/0 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784948971.957972:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001879354a. 00000400:00000001:3.0:1784948971.957976:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948971.957985:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4e2c75c. 00000020:00000010:0.0:1784948971.957986:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000005069617c. 00000400:00000200:3.0:1784948971.958000:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000020:00000010:0.0:1784948971.958002:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000010:3.0:1784948971.958008:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000100:00100000:2.0:1784948971.958010:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000008db39772 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289665152:12345-0@lo:400: Request processed in 621us (1215us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948971.958021:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784948971.958022:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00100000:2.0:1784948971.958031:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 746 00000100:00000001:0.0:1784948971.958032:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000200:3.0:1784948971.958036:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948971.958039:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:2.0:1784948971.958041:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948971.958047:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948971.958052:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948971.958061:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784948971.958064:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:3.0:1784948971.958070:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948971.958072:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000010:2.0:1784948971.958072:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000091201221. 00000100:00000001:0.0:1784948971.958079:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948971.958087:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784948971.958098:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e5260462. 00000100:00000001:3.0:1784948971.958102:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948971.958107:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948971.958108:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784948971.958109:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784948971.958120:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948971.958125:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:3.0:1784948971.958126:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000010:2.0:1784948971.958133:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c38711c7. 00000100:00000001:3.0:1784948971.958134:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948971.958139:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948971.958140:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948971.958143:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:2.0:1784948971.958145:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948971.958146:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:3.0:1784948971.958148:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948971.958150:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948971.958154:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784948971.958160:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=143 v=5 (5 5 5 5) 00000100:00100000:0.0:1784948971.958165:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 722 00000100:00000200:2.0:1784948971.958166:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:3.0:1784948971.958169:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=143 v=5 (1 1 1 1) 00000100:00000040:0.0:1784948971.958178:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000010:2.0:1784948971.958179:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000075224c9. 00000100:00000001:1.0:1784948971.958180:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.958180:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948971.958216:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554961536 : -115544154590080 : ffff96e9c7b5b480) 00000100:00000001:3.0:1784948971.958216:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948971.958217:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948971.958219:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000088cde038. 00010000:00000001:3.0:1784948971.958232:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000040:0.0:1784948971.958239:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ec3ca5ba x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:632/0 lens 224/0 e 0 to 0 dl 1784948982 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948971.958239:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.958249:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948971.958264:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948971.958266:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948971.958276:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784948971.958280:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948971.958283:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948971.958288:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:0.0:1784948971.958291:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784948971.958294:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948971.958302:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ec3ca5ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289665280:12345-192.168.202.137@tcp:400: 00000100:00000040:3.0:1784948971.958303:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948971.958322:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289665280 00000020:00000001:0.0:1784948971.958330:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:3.0:1784948971.958335:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289665152:0@lo:400: 00000020:00000001:0.0:1784948971.958337:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948971.958343:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.958347:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948971.958349:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784948971.958352:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784948971.958354:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000040:3.0:1784948971.958359:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb4f6eba x1871651289665152/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948971.958361:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948971.958368:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948971.958375:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784948971.958379:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784948971.958381:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:3.0:1784948971.958384:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:0.0:1784948971.958386:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948971.958390:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 00000020:00000001:0.0:1784948971.958392:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 02000000:00000001:3.0:1784948971.958400:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784948971.958401:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:3.0:1784948971.958407:0:337120:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:0.0:1784948971.958409:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000040:3.0:1784948971.958413:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 02000000:00000010:0.0:1784948971.958417:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000029365fd3. 00000020:00000001:3.0:1784948971.958419:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:0.0:1784948971.958423:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948971.958426:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 00000100:00000001:0.0:1784948971.958429:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948971.958436:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948971.958436:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948971.958442:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00010000:00000040:0.0:1784948971.958444:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289665280 00000100:00000010:3.0:1784948971.958448:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb4f6eba. 00010000:00000001:0.0:1784948971.958451:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948971.958457:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948971.958463:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:0.0:1784948971.958464:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ec3ca5ba x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:632/0 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948971.958468:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948971.958478:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948971.958483:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948971.958486:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948971.958488:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784948971.958493:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.958494:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948971.958505:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=138 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948971.958512:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948971.958523:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784948971.958526:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784948971.958529:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948971.958531:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948971.958534:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948971.958537:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784948971.958540:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000100:00000200:0.0:1784948971.958544:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289665280, offset 224 00000400:00000200:0.0:1784948971.958550:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948971.958558:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948971.958564:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3475:3475:233:4294967295] 192.168.202.137@tcp LPNI seq info [2233:2233:8:4294967295] 00000400:00000200:0.0:1784948971.958577:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784948971.958583:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784948971.958587:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f64a2f6c. 00000800:00000200:0.0:1784948971.958593:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784948971.958600:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784948971.958604:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f64a2f6c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784948971.958622:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948971.958626:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784948971.958629:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948971.958631:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948971.958633:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784948971.958637:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ec3ca5ba x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:632/0 lens 224/224 e 0 to 0 dl 1784948982 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948971.958648:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ec3ca5ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289665280:12345-192.168.202.137@tcp:400: Request processed in 382us (1022us total) trans 0 rc 0/0 00000100:00100000:0.0:1784948971.958657:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 722 00000100:00000040:0.0:1784948971.958660:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784948971.958663:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948971.958665:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784948971.958668:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000005069617c. 00000020:00000010:0.0:1784948971.958673:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784948971.958676:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001879354a. 00000020:00000040:0.0:1784948971.958680:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784948971.958683:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784948971.958845:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948971.958852:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f64a2f6c. 00000400:00000200:2.0:1784948971.958858:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948971.958866:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948971.958871:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:2.0:1784948971.958874:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000100:00000001:2.0:1784948971.958878:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948971.958880:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784948971.958887:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948971.958895:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948971.958898:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948971.958902:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948971.958955:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784948971.958968:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6438f00 00000400:00000200:2.0:1784948971.958976:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x89f1 [1] + 224 00000800:00000001:2.0:1784948971.958984:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948971.958993:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948971.958995:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948971.959000:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948971.959004:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948971.959009:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948971.959022:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948971.959043:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948971.959049:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948971.959052:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948971.959159:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948971.959167:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948971.959176:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948971.959179:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948971.959183:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000400:00000200:3.0:1784948971.959228:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:3.0:1784948971.959231:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:3.0:1784948971.959237:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948971.959248:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948971.959268:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948971.959284:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948971.959287:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948971.959292:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948971.959296:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948971.959298:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948971.959301:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948971.959306:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948971.959314:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=138 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948971.959321:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=138 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948971.959326:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948971.959328:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948971.959332:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948971.959338:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948971.959365:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.959370:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948971.959374:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948971.959377:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948971.959380:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948971.959386:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948971.959400:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948971.959404:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948971.959407:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948971.959413:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948971.959429:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb5afc66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289665280:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948971.959436:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948971.959441:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948971.959445:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb5afc66 x1871651289665280/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948971.959459:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948971.959462:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948971.959466:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 02000000:00000001:3.0:1784948971.959472:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948971.959475:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948971.959477:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948971.959482:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948971.959485:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 02000000:00000001:3.0:1784948971.959490:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948971.959493:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948971.959496:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb5afc66. 00000100:00000001:3.0:1784948971.959501:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948971.959503:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948971.959506:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948971.959514:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948971.959517:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948971.959520:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948971.959523:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948972.000013:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305911339, at 4305911340 00000100:00000010:0.0:1784948972.000028:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003e9ea21c. 00000100:00000040:1.0:1784948972.000032:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305911339, at 4305911340 00000100:00000010:1.0:1784948972.000048:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000040:2.0:1784948972.000054:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305911339, at 4305911340 00000100:00000010:2.0:1784948972.000070:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002b896c5a. 00000100:00000040:3.0:1784948972.000075:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305911339, at 4305911340 00000100:00000010:3.0:1784948972.000090:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000800:00000100:1.0:1784948972.148219:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.3@tcp 00000800:00000010:1.0:1784948972.148229:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008f5595bf. 00000400:00000200:1.0:1784948972.148242:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948972.148254:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948972.148266:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11245 health :900 00000400:00000200:1.0:1784948972.148275:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948972.148281:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:1.0:1784948972.148290:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000200:2.0:1784948972.148360:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948972.148374:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2861 00000400:00000001:2.0:1784948972.148383:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948972.148386:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948972.148389:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:2.0:1784948972.148392:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000400:00000200:2.0:1784948972.148398:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948972.148403:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:2.0:1784948972.148408:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009802f36f. 00000400:00000200:2.0:1784948972.148416:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-110 00000400:00000200:2.0:1784948972.148420:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -110 00000400:00000200:2.0:1784948972.148425:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00010000:00000001:2.0:1784948972.788060:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948972.788069:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948972.788079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948972.788081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948972.788268:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948972.788276:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948972.788282:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948972.980039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948972.980054:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948972.980057:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948972.980063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948972.980067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948972.980071:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948972.980073:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948973.172068:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784948973.172079:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000065a1450d. 00000800:00000100:1.0:1784948973.172092:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000400:00000010:3.0:1784948973.172093:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006cd4526f. 00000400:00000010:3.0:1784948973.172105:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000800:00000010:1.0:1784948973.172106:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d104522b. 00000400:00000010:3.0:1784948973.172120:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000092f0a178. 00000400:00000200:1.0:1784948973.172121:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948973.172135:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948973.172143:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948973.172164:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948973.172198:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948973.172205:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3476:3476:234:4294967295] 192.168.202.137@tcp LPNI seq info [2234:2234:8:4294967295] 00000400:00000200:3.0:1784948973.172214:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948973.172224:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948973.172231:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948973.172236:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11246 health :800 00000800:00000010:3.0:1784948973.172242:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000400:00000200:1.0:1784948973.172247:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948973.172256:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000800:00000200:3.0:1784948973.172264:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784948973.172272:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000800:00000200:3.0:1784948973.172283:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948973.172290:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000800:00000100:1.0:1784948973.172324:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.7@tcp 00000800:00000010:1.0:1784948973.172329:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000034aacccc. 00000400:00000200:1.0:1784948973.172336:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784948973.172344:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784948973.172347:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948973.172359:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948973.172362:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2861 00000400:00000200:1.0:1784948973.172370:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000001:2.0:1784948973.172376:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1784948973.172383:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000001:2.0:1784948973.172384:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948973.172392:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784948973.172397:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000400:00000200:2.0:1784948973.172404:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948973.172413:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:2.0:1784948973.172421:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000400:00000200:2.0:1784948973.172429:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:2.0:1784948973.172434:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:2.0:1784948973.172441:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000400:00000200:2.0:1784948973.172447:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2861 00000400:00000001:2.0:1784948973.172451:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948973.172452:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948973.172454:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000400:00000010:2.0:1784948973.172457:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000400:00000200:2.0:1784948973.172460:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948973.172463:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:2.0:1784948973.172466:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf0e6e18. 00000400:00000200:2.0:1784948973.172470:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-110 00000400:00000200:2.0:1784948973.172472:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -110 00000400:00000200:2.0:1784948973.172476:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000800:00000200:0.0:1784948973.172503:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948973.172512:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:0.0:1784948973.172522:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948973.172532:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948973.172533:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784948973.172544:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948973.172549:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784948973.172554:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948973.172561:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948973.172570:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948973.172580:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948973.172597:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948973.172608:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948973.172619:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948973.172625:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3477:3477:236:4294967295] 192.168.202.137@tcp LPNI seq info [2235:2235:8:4294967295] 00000400:00000200:1.0:1784948973.172636:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948973.172642:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948973.172647:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000800:00000200:1.0:1784948973.172653:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948973.172659:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948973.172663:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784948973.172724:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948973.172828:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948973.172835:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000400:00000200:1.0:1784948973.172843:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948973.172846:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948973.172855:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948973.172863:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948973.172869:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948973.172873:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948973.172880:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948973.172894:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948973.172947:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8a05 00000800:00000001:0.0:1784948973.172954:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948973.172967:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948973.172970:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948973.172974:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948973.172980:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:0.0:1784948973.172983:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000400:00000200:0.0:1784948973.172986:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948973.172989:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000065a1450d. 00000400:00000010:0.0:1784948973.172993:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006cd4526f. 00000800:00000001:0.0:1784948973.173001:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948973.173003:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948973.812023:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948973.812042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948973.812049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948973.812050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948973.812117:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948973.812121:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948973.812124:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948974.004021:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948974.004042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948974.004046:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948974.004054:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948974.004059:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948974.004064:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948974.004069:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784948974.196204:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.6@tcp 00000800:00000010:1.0:1784948974.196225:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000062b4a223. 00000400:00000200:1.0:1784948974.196236:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948974.196243:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948974.196252:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11247 health :900 00000400:00000200:1.0:1784948974.196259:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948974.196264:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:1.0:1784948974.196269:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000800:00000100:1.0:1784948974.196299:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.11@tcp 00000800:00000010:1.0:1784948974.196302:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e6aac0b. 00000400:00000200:1.0:1784948974.196309:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784948974.196314:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948974.196319:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784948974.196324:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:1.0:1784948974.196330:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:2.0:1784948974.196353:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784948974.196366:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2861 00000400:00000001:2.0:1784948974.196376:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948974.196380:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948974.196384:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:2.0:1784948974.196387:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000400:00000200:2.0:1784948974.196393:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948974.196398:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:2.0:1784948974.196404:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001a4eba99. 00000400:00000200:2.0:1784948974.196413:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-110 00000400:00000200:2.0:1784948974.196416:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -110 00000400:00000200:2.0:1784948974.196421:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:2.0:1784948974.196425:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2861 00000400:00000001:2.0:1784948974.196428:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948974.196430:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784948974.196432:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784948974.196433:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000400:00000200:2.0:1784948974.196436:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784948974.196438:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:2.0:1784948974.196440:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032fe57fd. 00000400:00000200:2.0:1784948974.196444:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:2.0:1784948974.196448:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:2.0:1784948974.196451:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00010000:00000001:2.0:1784948974.835979:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948974.835985:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948974.835990:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948974.835991:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948974.836048:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948974.836054:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948974.836058:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948975.028041:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948975.028063:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948975.028066:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948975.028075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948975.028079:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948975.028084:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948975.028089:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948975.220024:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784948975.220037:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000006fb16088. 00000400:00000010:3.0:1784948975.220052:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006cd4526f. 00000400:00000010:3.0:1784948975.220059:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000400:00000010:3.0:1784948975.220072:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000890c5791. 00000400:00000200:3.0:1784948975.220081:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948975.220099:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948975.220107:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948975.220112:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3478:3478:238:4294967295] 192.168.202.137@tcp LPNI seq info [2236:2236:8:4294967295] 00000400:00000200:3.0:1784948975.220120:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948975.220129:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948975.220136:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948975.220142:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948975.220150:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948975.220204:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948975.220210:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784948975.220234:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11249 health :800 00000800:00000200:1.0:1784948975.220394:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948975.220403:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:1.0:1784948975.220414:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948975.220425:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784948975.220433:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:2.0:1784948975.220437:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948975.220439:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784948975.220461:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948975.220468:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948975.220477:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948975.220488:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784948975.220506:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784948975.220517:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784948975.220529:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948975.220535:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3479:3479:238:4294967295] 192.168.202.137@tcp LPNI seq info [2237:2237:8:4294967295] 00000400:00000200:2.0:1784948975.220547:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784948975.220553:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948975.220558:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000093457b1b. 00000800:00000200:2.0:1784948975.220565:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948975.220572:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948975.220576:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000093457b1b type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948975.220705:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948975.220744:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784948975.220753:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784948975.220754:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000093457b1b. 00000800:00000001:1.0:1784948975.220767:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948975.220768:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948975.220769:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784948975.220781:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948975.220789:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784948975.220793:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948975.220809:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784948975.220827:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8a09 00000800:00000001:1.0:1784948975.220835:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948975.220845:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948975.220848:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948975.220853:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784948975.220859:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000400:00000010:1.0:1784948975.220862:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000400:00000200:1.0:1784948975.220866:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784948975.220869:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000006fb16088. 00000400:00000010:1.0:1784948975.220873:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006cd4526f. 00000800:00000001:1.0:1784948975.220881:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948975.220884:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948975.859976:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948975.859983:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948975.859988:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948975.859989:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948975.860123:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948975.860127:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948975.860130:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784948976.052050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784948976.052059:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784948976.052079:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948951/real 1784948951] req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948976 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784948976.052110:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948976.052114:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948976.052120:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009ef01e88. 00000400:00000200:2.0:1784948976.052131:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000400:00000010:2.0:1784948976.052134:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 00000100:00000001:2.0:1784948976.052140:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948976.052145:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948976 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948976.052182:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948976 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948976.052216:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948976.052218:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.052220:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948976.052222:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784948976.052225:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948976 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948976.052230:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.052237:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948976.052238:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948976.052240:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948976.052242:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948976.052244:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948976.052247:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948976 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948976.052252:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784948976.052254:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.052256:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948976.052256:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.052258:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784948976.052260:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784948976.052281:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784948976.052306:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:2.0:1784948976.052310:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784948976.052315:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948976 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948976.052325:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e8f14423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289664128:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784948976.052331:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.052334:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948976.052337:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e8f14423 x1871651289664128/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948976 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948976.052345:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948976.052348:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948976.052369:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d8bbc1a8. 02000000:00000001:2.0:1784948976.052373:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948976.052375:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948976.052377:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784948976.052379:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948976.052381:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000b91d4ef5. 02000000:00000001:2.0:1784948976.052385:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948976.052386:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948976.052388:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e8f14423. 00000100:00000001:2.0:1784948976.052392:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948976.052393:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.052394:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.052397:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948976.052398:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948976.052400:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948976.052402:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948976.052403:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948976.052410:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948976.052416:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000020:00000040:0.0:1784948976.052420:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948976.052424:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948976.052426:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948976.052431:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948976.052437:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b84187a1. 00000100:00000001:0.0:1784948976.052442:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948976.052447:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948976.052455:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948976.052457:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948976.052461:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_00_03+3] req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948976.052482:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948976.052484:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948976.052486:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784948976.052489:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784948976.052490:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948976.052493:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784948976.052497:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11223 00000400:00000200:0.0:1784948976.052505:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784948976.052509:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11222 00000100:00000001:1.0:1784948976.052516:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784948976.052521:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:1.0:1784948976.052522:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948976.052525:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:1.0:1784948976.052526:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784948976.052529:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000040:1.0:1784948976.052533:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948976.052534:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948976.052539:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784948976.052544:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000001:0.0:1784948976.052550:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784948976.052551:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1784948976.052553:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000200:2.0:1784948976.052553:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:1.0:1784948976.052554:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948976.052556:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784948976.052557:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948976.052562:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1784948976.052563:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000200:2.0:1784948976.052563:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000001:3.0:1784948976.052563:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948976.052567:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784948976.052571:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784948976.052572:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948976.052576:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:2.0:1784948976.052576:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e5260462. 00000100:00100000:1.0:1784948976.052579:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289665408:0@lo:400: 00000400:00000010:2.0:1784948976.052584:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000040:0.0:1784948976.052586:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948976.052586:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784948976.052590:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784948976.052592:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 02000000:00000001:1.0:1784948976.052593:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:2.0:1784948976.052593:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000064d54234. 00000100:00000001:0.0:1784948976.052594:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:1.0:1784948976.052598:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948976.052600:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:1.0:1784948976.052603:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784948976.052604:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00080000:0.0:1784948976.052608:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000010:1.0:1784948976.052609:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000072ce5ef0. 00000100:00080000:0.0:1784948976.052614:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 02000000:00000001:1.0:1784948976.052615:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.052618:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948976.052620:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 02000000:00000001:0.0:1784948976.052622:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1784948976.052623:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000057b2318d. 00000400:00000200:2.0:1784948976.052629:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784948976.052630:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784948976.052631:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000200:2.0:1784948976.052635:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:0.0:1784948976.052638:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948976.052640:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289665408, portal 25 00000400:00000200:2.0:1784948976.052642:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784948976.052643:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1784948976.052644:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948976.052647:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.052648:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 00000400:00000200:2.0:1784948976.052650:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000010:0.0:1784948976.052657:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000073a82e6. 00000100:00000040:1.0:1784948976.052660:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948976.052660:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3480:3480:238:4294967295] 1.2.3.4@tcp LPNI seq info [104:104:8:4294967295] 00000020:00000040:0.0:1784948976.052662:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784948976.052670:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784948976.052674:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948976.052677:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000001:0.0:1784948976.052680:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784948976.052681:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784948976.052684:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948976.052689:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948976.052689:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:2.0:1784948976.052694:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 02000000:00000010:0.0:1784948976.052697:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006296bcc7. 00000400:00000010:1.0:1784948976.052697:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000800:00000200:2.0:1784948976.052703:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000200:1.0:1784948976.052704:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289665408, offset 0 00000100:00000001:0.0:1784948976.052705:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784948976.052711:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000093457b1b. 00000100:00100000:0.0:1784948976.052715:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784948976.052718:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000200:2.0:1784948976.052725:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.0:1784948976.052732:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 31 secs later) 00000100:00000001:0.0:1784948976.052733:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1784948976.052734:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948976.052737:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1784948976.052745:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:0.0:1784948976.052746:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000073a82e6] to pc [ptlrpcd_rcv+-1] req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948976.052751:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6438f80 00000800:00000200:2.1:1784948976.052753:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 31 secs later) 00000800:00000100:2.0:1784948976.052766:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000400:00000200:1.0:1784948976.052767:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x7ff9 [2] + 0 00000800:00000010:2.0:1784948976.052771:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000093457b1b. 00000400:00000010:1.0:1784948976.052773:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000093aab5da. 00000100:00000001:0.0:1784948976.052776:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948976.052778:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784948976.052779:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000009412aeee 00000100:00000040:0.0:1784948976.052780:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948976.052786:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:1.0:1784948976.052790:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:0.0:1784948976.052791:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000df8ddd96. 00000400:00000200:2.0:1784948976.052791:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000001:3.0:1784948976.052795:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784948976.052798:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:2.0:1784948976.052800:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000001:3.0:1784948976.052801:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784948976.052802:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948976.052806:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948976.052807:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784948976.052808:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:1.0:1784948976.052809:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009412aeee 02000000:00000001:0.0:1784948976.052812:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948976.052815:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948976.052815:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000009412aeee. 00000100:00000040:3.0:1784948976.052816:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948976.052819:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:1.0:1784948976.052822:0:337122:0:(events.c:286:request_in_callback()) Process entered 02000000:00000010:0.0:1784948976.052823:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a3cf4688. 00000100:00000200:1.0:1784948976.052826:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784948976.052829:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948976.052831:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:1.0:1784948976.052834:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000875035e8 x1871651289665408 msgsize 224 00000100:00000040:0.0:1784948976.052839:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784948976.052840:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00100000:1.0:1784948976.052851:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:2.0:1784948976.052853:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000100:00000001:3.0:1784948976.052854:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784948976.052858:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784948976.052859:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784948976.052861:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948976.052865:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1784948976.052866:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784948976.052869:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000100:00000001:0.0:1784948976.052870:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1784948976.052871:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948976.052872:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:2.0:1784948976.052875:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00000001:0.0:1784948976.052876:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948976.052877:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948976.052883:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784948976.052884:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000100:00100000:3.0:1784948976.052886:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000073a82e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289665536:1.2.3.4@tcp:8: 00000100:00000040:0.0:1784948976.052888:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000df8ddd96] to pc [ptlrpcd_00_00+0] req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784948976.052889:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000400:00000200:2.0:1784948976.052893:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:3.0:1784948976.052895:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784948976.052898:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948976.052900:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948976.052902:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e5260462. 00000400:00000200:2.0:1784948976.052932:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:3.0:1784948976.052937:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948976.052938:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948976.052938:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:3.0:1784948976.052943:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.052947:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948976.052949:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:3.0:1784948976.052949:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948976.052957:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11253) 00000100:00000001:3.0:1784948976.052959:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948976.052961:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784948976.052963:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948976.052967:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948976.052968:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948976.052971:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948976.052972:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948976.052974:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.052977:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948976.052979:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.052981:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948976.052982:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948976.052989:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.052990:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948976.052992:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948976.052994:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948976.053001:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784948976.053002:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053003:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948976.053007:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948976.053008:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305915435 02000000:00000001:0.0:1784948976.053010:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784948976.053013:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784948976.053014:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00100000:0.0:1784948976.053017:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000df8ddd96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289665664:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948976.053019:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948976.053023:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784948976.053024:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1784948976.053025:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289665408 02000000:00000001:3.0:1784948976.053029:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:0.0:1784948976.053030:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948976.053031:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948976.053032:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305915435 02000000:00000001:0.0:1784948976.053033:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948976.053036:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948976.053039:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1784948976.053040:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 02000000:00000001:3.0:1784948976.053041:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948976.053043:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 00000100:00000001:1.0:1784948976.053046:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1784948976.053048:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948976.053048:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289665408 00000100:00000001:1.0:1784948976.053050:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305915435 00000020:00000001:3.0:1784948976.053053:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948976.053056:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000010:0.0:1784948976.053057:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000968c2fa4. 00000100:00000010:1.0:1784948976.053057:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000001:3.0:1784948976.053058:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784948976.053062:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784948976.053064:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000001:3.0:1784948976.053065:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000010:0.0:1784948976.053066:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000020:00000001:3.0:1784948976.053071:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000200:0.0:1784948976.053072:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289665664, portal 10 00000100:00000001:3.0:1784948976.053075:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948976.053076:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948976.053077:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305915435 00000100:00000001:0.0:1784948976.053080:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554970752 : -115544154580864 : ffff96e9c7b5d880) 00000020:00000010:3.0:1784948976.053081:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000226948e6. 00000100:00000010:1.0:1784948976.053083:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:3.0:1784948976.053088:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948976.053090:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000010:3.0:1784948976.053092:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000226948e6. 00000100:00000040:0.0:1784948976.053093:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948976.053095:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948976.053102:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948976.053102:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006cd4526f. 00000100:00000001:1.0:1784948976.053109:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053110:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:3.0:1784948976.053111:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000d6164f7. 00000100:00000001:1.0:1784948976.053112:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948976.053117:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000010:3.0:1784948976.053120:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000010:0.0:1784948976.053121:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000001:1.0:1784948976.053121:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948976.053125:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000200:0.0:1784948976.053126:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289665664, offset 0 00000100:00000001:1.0:1784948976.053129:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948976.053130:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784948976.053133:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784948976.053135:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948976.053136:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948976.053137:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948976.053140:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948976.053142:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948976.053143:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948976.053145:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784948976.053168:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948976.053168:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948976.053175:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948976.053178:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948976.053179:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784948976.053185:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784948976.053188:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3481:3481:238:4294967295] 192.168.202.137@tcp LPNI seq info [2238:2238:8:4294967295] 00000100:00000001:3.0:1784948976.053192:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948976.053195:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948976.053197:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948976.053202:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784948976.053206:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 747 00000400:00000200:0.0:1784948976.053207:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:3.0:1784948976.053213:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948976.053214:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948976.053220:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948976.053220:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466827856 : -115544242723760 : ffff96e9c274e450) 00000100:00000001:1.0:1784948976.053221:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948976.053228:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000595fa8f8. 00000100:00000040:3.0:1784948976.053232:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000875035e8 x1871651289665408/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:637/0 lens 224/0 e 0 to 0 dl 1784948987 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784948976.053238:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784948976.053247:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:0.0:1784948976.053251:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000010:3.0:1784948976.053253:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000226948e6. 00000800:00000200:0.0:1784948976.053255:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000595fa8f8 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948976.053257:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948976.053262:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000226948e6. 00000100:00100000:3.0:1784948976.053266:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000875035e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289665408:12345-0@lo:400: 00000100:00000200:3.0:1784948976.053269:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289665408 00000020:00000001:3.0:1784948976.053272:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784948976.053273:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948976.053275:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:0.0:1784948976.053278:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2121 usecs (3 jiffies) 00000020:00000001:3.0:1784948976.053278:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948976.053284:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784948976.053285:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305915396 00000020:00000001:3.0:1784948976.053287:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784948976.053289:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305915435 00000020:00000001:3.0:1784948976.053292:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784948976.053293:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305915396 00000020:00000001:3.0:1784948976.053296:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.053297:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305915435 00000020:00000001:3.0:1784948976.053301:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784948976.053302:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305915396 00000100:00000001:0.0:1784948976.053306:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305915435 00000020:00000001:3.0:1784948976.053306:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948976.053310:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053311:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305915396 00000100:00000001:0.0:1784948976.053315:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305915435 00000020:00000001:3.0:1784948976.053315:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784948976.053318:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784948976.053319:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948976.053322:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784948976.053323:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053326:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053328:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784948976.053328:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008d04782c. 00000100:00000001:0.0:1784948976.053332:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784948976.053332:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053336:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948976.053336:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948976.053340:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053341:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948976.053345:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000040:3.0:1784948976.053345:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289665408 00000100:00000001:0.0:1784948976.053347:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948976.053349:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784948976.053351:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784948976.053354:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000875035e8 x1871651289665408/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:637/0 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948976.053361:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948976.053363:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948976.053365:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=148 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948976.053368:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948976.053371:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784948976.053373:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784948976.053375:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948976.053376:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948976.053378:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948976.053380:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784948976.053382:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000200:3.0:1784948976.053385:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289665408, offset 224 00000400:00000200:3.0:1784948976.053390:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000200:1.0:1784948976.053395:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1784948976.053396:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:1.0:1784948976.053401:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000595fa8f8. 00000400:00000200:1.0:1784948976.053406:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948976.053406:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6438f80 00000400:00000200:3.0:1784948976.053415:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8a11 [1] + 224 00000400:00000200:1.0:1784948976.053416:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948976.053422:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948976.053423:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000400:00000010:1.0:1784948976.053427:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000400:00000200:3.0:1784948976.053431:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948976.053432:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784948976.053435:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948976.053439:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948976.053441:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948976.053460:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948976.053469:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948976.053472:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948976.053474:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948976.053477:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784948976.053481:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948976.053486:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948976.053500:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784948976.053504:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.053508:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948976.053510:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784948976.053514:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439080 00000400:00000200:2.0:1784948976.053522:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 162992 00000400:00000200:3.0:1784948976.053526:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000800:00000001:2.0:1784948976.053528:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784948976.053530:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00000001:3.0:1784948976.053534:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784948976.053535:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948976.053536:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948976.053540:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948976.053543:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948976.053544:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948976.053547:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948976.053548:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784948976.053551:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:0.0:1784948976.053555:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000001:3.0:1784948976.053555:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:0.0:1784948976.053560:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1784948976.053561:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784948976.053566:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000057b2318d. 00000020:00000001:3.0:1784948976.053566:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948976.053571:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000875035e8 x1871651289665408/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:637/0 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948976.053572:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000010:0.0:1784948976.053576:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000100:00000001:0.0:1784948976.053581:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784948976.053583:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00100000:3.0:1784948976.053586:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000875035e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289665408:12345-0@lo:400: Request processed in 336us (749us total) trans 0 rc 0/0 00000100:00000200:0.0:1784948976.053588:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948976.053588:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:2.0:1784948976.053596:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b8226094. 00000100:00000040:2.0:1784948976.053602:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000b8226094 x1871651289665664 msgsize 224 00000100:00100000:2.0:1784948976.053619:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00100000:3.0:1784948976.053620:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 747 00000100:00000200:0.0:1784948976.053621:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948976.053627:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784948976.053632:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784948976.053634:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948976.053635:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784948976.053638:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948976.053639:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948976.053653:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:3.0:1784948976.053653:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000d6164f7. 02000000:00000001:0.0:1784948976.053658:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000800:00000001:2.0:1784948976.053659:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784948976.053659:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:0.0:1784948976.053662:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000001:2.0:1784948976.053665:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.053666:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784948976.053666:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006cd4526f. 02000000:00000001:0.0:1784948976.053692:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784948976.053695:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:0.0:1784948976.053698:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=148 v=5 (5 5 5 5) 02000000:00000001:3.0:1784948976.053704:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:0.0:1784948976.053707:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=148 v=5 (1 1 1 1) 02000000:00000010:3.0:1784948976.053710:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008d04782c. 00000100:00000001:0.0:1784948976.053715:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000001:3.0:1784948976.053718:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784948976.053721:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.053721:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784948976.053723:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784948976.053729:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289665664 00010000:00000001:0.0:1784948976.053731:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:3.0:1784948976.053734:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:1.0:1784948976.053737:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00010000:00000001:0.0:1784948976.053739:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948976.053744:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4db7024. 00000100:00000001:1.0:1784948976.053745:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948976.053746:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.053753:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948976.053757:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006a025321. 00000100:00000040:0.0:1784948976.053759:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948976.053762:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948976.053773:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289665664 00000020:00000001:1.0:1784948976.053780:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784948976.053782:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784948976.053785:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784948976.053787:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1784948976.053790:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948976.053792:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784948976.053798:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:0.0:1784948976.053801:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948976.053805:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948976.053813:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948976.053822:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:0.0:1784948976.053831:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289665408:0@lo:400: 00000020:00000010:1.0:1784948976.053831:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004204c760. 00000100:00000001:1.0:1784948976.053841:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948976.053844:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948976.053847:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004204c760. 00000100:00000001:0.0:1784948976.053849:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948976.053855:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289665408/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948976.053856:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dd0652fb. 00000020:00000010:1.0:1784948976.053868:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004589cdd0. 00000100:00000001:0.0:1784948976.053872:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948976.053878:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:1.0:1784948976.053878:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 02000000:00000010:0.0:1784948976.053883:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000072ce5ef0. 02000000:00000001:0.0:1784948976.053892:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:1.0:1784948976.053895:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:0.0:1784948976.053896:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948976.053900:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784948976.053905:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948976.053907:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b84187a1. 02000000:00000001:0.0:1784948976.053911:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948976.053913:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948976.053917:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 00000100:00000001:1.0:1784948976.053934:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948976.053939:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948976.053943:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948976.053950:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784948976.053953:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948976.053955:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.053969:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948976.053982:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.053984:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.053991:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.053998:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948976.054000:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.054004:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948976.054005:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948976.054011:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948976.054013:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948976.054024:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 723 00000100:00000040:1.0:1784948976.054030:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948976.054034:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529470022400 : -115544239529216 : ffff96e9c2a5a300) 00000100:00000040:1.0:1784948976.054041:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b8226094 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:637/0 lens 224/0 e 0 to 0 dl 1784948987 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948976.054053:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948976.054057:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004204c760. 00000100:00000001:1.0:1784948976.054060:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948976.054062:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004204c760. 00000100:00100000:1.0:1784948976.054068:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b8226094 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289665664:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784948976.054073:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289665664 00000020:00000001:1.0:1784948976.054075:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948976.054079:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948976.054081:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948976.054083:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948976.054085:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784948976.054089:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948976.054091:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948976.054094:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948976.054096:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948976.054098:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948976.054100:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784948976.054104:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948976.054106:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948976.054110:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000feeafc8d. 02000000:00000001:1.0:1784948976.054113:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.054116:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948976.054118:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948976.054120:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289665664 00010000:00000001:1.0:1784948976.054123:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948976.054127:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b8226094 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:637/0 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948976.054136:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948976.054139:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948976.054141:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=143 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948976.054146:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948976.054149:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948976.054177:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948976.054181:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948976.054184:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948976.054187:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948976.054190:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948976.054193:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000100:00000200:1.0:1784948976.054197:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289665664, offset 224 00000400:00000200:1.0:1784948976.054203:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948976.054212:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948976.054219:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3482:3482:238:4294967295] 192.168.202.137@tcp LPNI seq info [2239:2239:8:4294967295] 00000400:00000200:1.0:1784948976.054230:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948976.054236:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948976.054240:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000800:00000200:1.0:1784948976.054246:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948976.054252:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948976.054256:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948976.054273:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948976.054277:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948976.054280:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948976.054281:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948976.054284:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948976.054288:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b8226094 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:637/0 lens 224/224 e 0 to 0 dl 1784948987 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948976.054300:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b8226094 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289665664:12345-192.168.202.137@tcp:400: Request processed in 246us (696us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948976.054312:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 723 00000100:00000040:1.0:1784948976.054316:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948976.054319:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948976.054323:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948976.054326:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004589cdd0. 00000020:00000010:1.0:1784948976.054330:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948976.054334:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dd0652fb. 00000020:00000040:1.0:1784948976.054340:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948976.054344:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948976.054430:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784948976.054431:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784948976.054439:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000800:00000001:1.0:1784948976.054445:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948976.054446:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948976.054457:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948976.054464:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784948976.054466:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948976.054473:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000400:00000010:0.0:1784948976.054480:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000400:00000200:1.0:1784948976.054482:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948976.054492:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948976.054496:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948976.054502:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439080 00000400:00000200:1.0:1784948976.054510:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8a19 [1] + 224 00000800:00000001:1.0:1784948976.054517:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948976.054528:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948976.054532:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948976.054537:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948976.054542:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948976.054546:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948976.054557:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948976.054578:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948976.054584:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948976.054587:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.054625:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948976.054629:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784948976.054634:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948976.054635:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784948976.054638:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000968c2fa4. 00000400:00000200:0.0:1784948976.054643:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:0.0:1784948976.054645:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000100:00000001:0.0:1784948976.054649:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948976.054655:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784948976.054674:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948976.054682:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784948976.054684:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.054686:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784948976.054688:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784948976.054690:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948976.054692:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948976.054694:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948976.054700:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=143 v=5 (5 5 5 5) 00000100:00001000:0.0:1784948976.054703:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=143 v=5 (1 1 1 1) 00000100:00000001:0.0:1784948976.054706:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784948976.054707:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784948976.054710:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784948976.054713:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948976.054722:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.054725:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784948976.054727:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784948976.054729:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784948976.054730:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948976.054733:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948976.054741:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948976.054743:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784948976.054745:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948976.054749:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948976.054757:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000df8ddd96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289665664:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784948976.054762:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.054764:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784948976.054767:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000df8ddd96 x1871651289665664/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948992 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948976.054775:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784948976.054777:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784948976.054779:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 02000000:00000001:0.0:1784948976.054782:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784948976.054784:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784948976.054785:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784948976.054788:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784948976.054790:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a3cf4688. 02000000:00000001:0.0:1784948976.054793:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784948976.054794:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784948976.054796:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000df8ddd96. 00000100:00000001:0.0:1784948976.054801:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784948976.054803:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.054805:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948976.054809:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948976.054810:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948976.054812:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948976.054815:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948976.096043:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305915435, at 4305915436 00000100:00000040:3.0:1784948976.096049:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305915435, at 4305915436 00000100:00000010:1.0:1784948976.096057:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:3.0:1784948976.096064:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:0.0:1784948976.096082:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305915435, at 4305915436 00000100:00000040:2.0:1784948976.096086:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305915435, at 4305915436 00000100:00000010:0.0:1784948976.096100:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:2.0:1784948976.096105:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784948976.884027:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948976.884036:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948976.884043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948976.884045:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948976.884191:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948976.884197:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948976.884201:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948977.076046:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948977.076055:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1784948977.076072:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784948976/real 0] req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948977 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.076091:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.076096:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948977.076098:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948977.076104:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948977 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.076115:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.076125:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948977.076129:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.076132:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948977.076136:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948977.076140:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948977.076175:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948977 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.076186:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1784948977.076192:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.076195:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948977.076197:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.076200:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1784948977.076203:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1784948977.076207:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1784948977.076241:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:3.0:1784948977.076246:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1784948977.076251:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948977 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948977.076263:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000073a82e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289665536:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784948977.076271:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.076276:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948977.076281:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000073a82e6 x1871651289665536/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784948977 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.076293:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1784948977.076298:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948977.076301:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:3.0:1784948977.076305:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948977.076310:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000006296bcc7. 02000000:00000001:3.0:1784948977.076318:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948977.076322:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948977.076324:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000073a82e6. 00000100:00000040:0.0:1784948977.076339:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948977.076343:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948977.076349:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948977.076353:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948977.076355:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.076361:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948977.076363:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948977.076364:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000df8ddd96. 00000100:00000001:3.0:1784948977.076370:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784948977.076373:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784948977.076375:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948977.076385:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948977.076390:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948977.076396:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948977.076407:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a3cf4688. 00000100:00000001:0.0:1784948977.076417:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948977.076425:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948977.076439:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948977.076443:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948977.076452:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000df8ddd96] to pc [ptlrpcd_00_01+1] req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948977.076495:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948977.076499:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948977.076503:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784948977.076507:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784948977.076510:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784948977.076515:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784948977.076522:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11223 00000100:00000001:2.0:1784948977.076525:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948977.076533:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948977.076536:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784948977.076538:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784948977.076546:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11248 00000100:00000001:3.0:1784948977.076550:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948977.076551:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.076559:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948977.076570:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:2.0:1784948977.076582:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784948977.076583:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:2.0:1784948977.076589:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948977.076591:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1784948977.076596:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784948977.076603:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000400:00000200:3.0:1784948977.076604:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 02000000:00000001:2.0:1784948977.076605:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948977.076616:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948977.076618:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1784948977.076620:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00100000:2.0:1784948977.076630:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000df8ddd96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289665792:0@lo:400: 00000400:00000010:3.0:1784948977.076635:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006cd4526f. 00000100:00000001:2.0:1784948977.076642:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:3.0:1784948977.076647:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000040:0.0:1784948977.076652:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:2.0:1784948977.076655:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:3.0:1784948977.076660:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000005df51d2. 00000100:00000001:0.0:1784948977.076665:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948977.076665:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948977.076675:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 02000000:00000001:2.0:1784948977.076680:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1784948977.076681:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948977.076689:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000010:2.0:1784948977.076695:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 00000020:00000001:0.0:1784948977.076696:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784948977.076697:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000020:00000040:0.0:1784948977.076707:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:2.0:1784948977.076708:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948977.076713:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1784948977.076722:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:2.0:1784948977.076729:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d3f471e1. 00000400:00000200:3.0:1784948977.076729:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:0.0:1784948977.076732:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:3.0:1784948977.076743:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000010:2.0:1784948977.076748:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000040:0.0:1784948977.076749:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000400:00000200:3.0:1784948977.076754:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:0.0:1784948977.076765:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1784948977.076767:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289665792, portal 25 00000400:00000200:3.0:1784948977.076770:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3483:3483:238:4294967295] 1.2.3.4@tcp LPNI seq info [105:105:8:4294967295] 00000100:00000040:0.0:1784948977.076779:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784948977.076782:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948977.076788:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784948977.076790:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554970752 : -115544154580864 : ffff96e9c7b5d880) 00000400:00000200:3.0:1784948977.076791:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000020:00000040:0.0:1784948977.076796:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:0.0:1784948977.076813:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000040:2.0:1784948977.076816:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948977.076821:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00080000:0.0:1784948977.076833:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:0.0:1784948977.076852:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948977.076854:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 02000000:00000001:0.0:1784948977.076861:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:3.0:1784948977.076868:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000100:00000001:2.0:1784948977.076871:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948977.076882:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000200:3.0:1784948977.076891:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:2.0:1784948977.076893:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000800:00000200:3.0:1784948977.076897:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 30 secs later) 00000100:00000200:2.0:1784948977.076903:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289665792, offset 0 00000800:00000200:3.1:1784948977.076959:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:2.0:1784948977.076966:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000200:3.1:1784948977.076969:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 30 secs later) 00000800:00000100:3.0:1784948977.076981:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000800:00000010:3.0:1784948977.076989:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:2.0:1784948977.076992:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948977.076999:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784948977.077012:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784948977.077020:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439100 00000400:00000200:3.0:1784948977.077023:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948977.077032:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:2.0:1784948977.077037:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8015 [2] + 0 02000000:00000001:0.0:1784948977.077039:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784948977.077046:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000001:0.0:1784948977.077053:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948977.077054:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000603fb577. 00000100:00000001:0.0:1784948977.077063:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1784948977.077067:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000001113b68a 00000100:00000001:0.0:1784948977.077070:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948977.077078:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000400:00000200:2.0:1784948977.077080:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:0.0:1784948977.077084:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:3.0:1784948977.077089:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 02000000:00000001:0.0:1784948977.077093:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1784948977.077097:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784948977.077099:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948977.077103:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000001:0.0:1784948977.077111:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1784948977.077115:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001113b68a 02000000:00000001:0.0:1784948977.077116:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1784948977.077119:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000010:2.0:1784948977.077124:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000001113b68a. 02000000:00000001:0.0:1784948977.077126:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.077138:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000001:3.0:1784948977.077140:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:0.0:1784948977.077141:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006e175381. 00000100:00000200:2.0:1784948977.077170:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000001:3.0:1784948977.077175:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1784948977.077180:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000019a655f1 x1871651289665792 msgsize 224 00000100:00000001:0.0:1784948977.077181:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948977.077181:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:3.0:1784948977.077191:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000400:00000200:3.0:1784948977.077202:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00100000:0.0:1784948977.077203:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1784948977.077205:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948977.077214:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948977.077216:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:3.0:1784948977.077229:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006cd4526f. 00000100:00000001:2.0:1784948977.077237:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784948977.077241:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1784948977.077243:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:0.0:1784948977.077248:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948977.077253:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000400:00000200:2.0:1784948977.077254:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:2.0:1784948977.077263:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000100:00000040:0.0:1784948977.077265:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_rcv+-1] req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948977.077270:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:2.0:1784948977.077271:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784948977.077286:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948977.077307:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784948977.077311:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948977.077315:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.077319:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948977.077321:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:0.0:1784948977.077322:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784948977.077327:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1784948977.077332:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289665792 00000100:00000001:3.0:1784948977.077335:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784948977.077342:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948977.077347:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784948977.077349:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784948977.077351:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948977.077357:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948977.077360:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c1a2ac13. 00000100:00000001:2.0:1784948977.077361:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784948977.077373:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948977.077373:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784948977.077385:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948977.077387:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948977.077388:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948977.077391:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948977.077394:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948977.077398:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.077399:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305916459 02000000:00000001:3.0:1784948977.077403:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948977.077404:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948977.077410:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b84187a1. 02000000:00000001:3.0:1784948977.077413:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1784948977.077416:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289665792 00000100:00000010:2.0:1784948977.077417:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000100:00000001:0.0:1784948977.077422:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948977.077426:0:342757:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:3.0:1784948977.077427:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.077433:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000040:1.0:1784948977.077434:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:0.0:1784948977.077437:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1784948977.077441:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289665920:192.168.202.137@tcp:8: 00000020:00000001:1.0:1784948977.077443:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948977.077450:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305916459 00000020:00000040:1.0:1784948977.077451:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948977.077459:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1784948977.077464:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f4af3cce. 00000020:00000001:1.0:1784948977.077465:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:3.0:1784948977.077467:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948977.077476:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.077477:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1784948977.077478:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:3.0:1784948977.077487:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948977.077489:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948977.077491:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305916459 00000100:00000001:1.0:1784948977.077493:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948977.077495:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948977.077496:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000008d04782c. 00000100:00000010:2.0:1784948977.077502:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a3d0cc2f. 02000000:00000001:3.0:1784948977.077506:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.077511:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000040:0.0:1784948977.077512:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c1a2ac13] to pc [ptlrpcd_00_02+2] req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948977.077518:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000df521cee. 00000100:00000001:2.0:1784948977.077522:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305916459 00000400:00000010:3.0:1784948977.077530:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000010:2.0:1784948977.077537:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000020:00000010:1.0:1784948977.077540:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000200:3.0:1784948977.077542:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289665920, portal 4 00000100:00000001:2.0:1784948977.077555:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948977.077556:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948977.077556:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948977.077562:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 00000020:00000010:1.0:1784948977.077568:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:2.0:1784948977.077575:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948977.077579:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948977.077580:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dd0652fb. 00000100:00000040:3.0:1784948977.077581:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948977.077592:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948977.077595:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11254) 00000020:00000010:1.0:1784948977.077596:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004589cdd0. 00000100:00000001:2.0:1784948977.077608:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.077612:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:1.0:1784948977.077613:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000001:2.0:1784948977.077617:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948977.077621:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784948977.077627:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948977.077630:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948977.077631:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000010:3.0:1784948977.077635:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00000001:0.0:1784948977.077637:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948977.077638:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948977.077639:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000200:3.0:1784948977.077641:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289665920, offset 0 00000100:00000001:2.0:1784948977.077649:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784948977.077650:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:0.0:1784948977.077651:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948977.077657:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948977.077662:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948977.077665:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948977.077675:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.077675:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784948977.077681:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948977.077685:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948977.077697:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948977.077699:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1784948977.077699:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784948977.077700:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948977.077709:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948977.077713:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:0.0:1784948977.077722:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784948977.077722:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784948977.077724:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:0.0:1784948977.077733:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948977.077733:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784948977.077734:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:0.0:1784948977.077746:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948977.077752:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 748 00000400:00000200:3.0:1784948977.077753:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3484:3484:238:4294967295] 192.168.202.137@tcp LPNI seq info [2240:2240:8:4294967295] 00000100:00100000:0.0:1784948977.077764:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c1a2ac13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289666048:192.168.202.137@tcp:400: 00000100:00000040:1.0:1784948977.077765:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784948977.077774:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466821712 : -115544242729904 : ffff96e9c274cc50) 00000100:00000001:0.0:1784948977.077782:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948977.077789:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1784948977.077790:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000019a655f1 x1871651289665792/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:638/0 lens 224/0 e 0 to 0 dl 1784948988 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948977.077794:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:0.0:1784948977.077795:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.077804:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784948977.077806:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784948977.077816:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948977.077818:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 00000800:00000200:3.0:1784948977.077823:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948977.077827:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:0.0:1784948977.077830:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948977.077838:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000800:00000010:3.0:1784948977.077838:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000400:00000010:0.0:1784948977.077840:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000968c2fa4. 00000100:00000001:1.0:1784948977.077853:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784948977.077857:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000020:00000010:1.0:1784948977.077862:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000800:00000200:3.0:1784948977.077863:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784948977.077869:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666048, portal 10 00000100:00000001:0.0:1784948977.077875:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784948977.077881:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554978816 : -115544154572800 : ffff96e9c7b5f800) 00000100:00100000:1.0:1784948977.077882:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000019a655f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289665792:12345-0@lo:400: 00000800:00000200:3.0:1784948977.077890:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000200:1.0:1784948977.077899:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289665792 00000800:00000200:3.0:1784948977.077900:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 616 niov 1 nkiov 1 00000100:00000040:0.0:1784948977.077942:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948977.077954:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784948977.077961:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948977.077962:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1784948977.077964:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948977.077971:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948977.077973:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000040:0.0:1784948977.077974:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784948977.077982:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784948977.077984:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305916460 00000400:00000010:0.0:1784948977.077985:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000020:00000001:1.0:1784948977.077988:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000200:0.0:1784948977.077995:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289666048, offset 0 00000020:00000001:1.0:1784948977.077997:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784948977.078001:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305916459 00000020:00000001:1.0:1784948977.078003:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948977.078013:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784948977.078014:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784948977.078018:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305916460 00000020:00000001:1.0:1784948977.078021:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948977.078027:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.078029:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305916459 00000020:00000001:1.0:1784948977.078031:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:0.0:1784948977.078035:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784948977.078039:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948977.078042:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305916460 02000000:00000001:1.0:1784948977.078046:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:0.0:1784948977.078047:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948977.078051:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305916459 02000000:00000010:1.0:1784948977.078057:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000057fac1f9. 00000400:00000200:0.0:1784948977.078060:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784948977.078062:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305916460 02000000:00000001:1.0:1784948977.078066:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948977.078073:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784948977.078075:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.078080:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305916459 00000400:00000200:0.0:1784948977.078083:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:1.0:1784948977.078084:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.078088:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000040:1.0:1784948977.078094:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289665792 00000400:00000200:0.0:1784948977.078098:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3485:3485:237:4294967295] 192.168.202.137@tcp LPNI seq info [2241:2241:7:4294967295] 00000100:00000001:3.0:1784948977.078099:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948977.078103:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948977.078110:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:1.0:1784948977.078118:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000019a655f1 x1871651289665792/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:638/0 lens 224/224 e 0 to 0 dl 1784948988 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.078118:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.078132:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1784948977.078139:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784948977.078140:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1784948977.078173:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784948977.078174:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1784948977.078182:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.078182:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:0.0:1784948977.078183:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948977.078189:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948977.078193:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=149 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948977.078195:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948977.078196:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008746b1e6. 00000100:00000001:1.0:1784948977.078210:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1784948977.078212:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000200:0.0:1784948977.078216:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:1.0:1784948977.078219:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000010:2.0:1784948977.078225:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000100:00000001:1.0:1784948977.078227:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000200:0.0:1784948977.078237:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000001:3.0:1784948977.078239:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784948977.078240:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:2.0:1784948977.078243:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784948977.078248:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948977.078249:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008746b1e6 type 1, nob 320 niov 1 nkiov 1 00000800:00000001:3.0:1784948977.078259:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948977.078261:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784948977.078267:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948977.078269:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784948977.078277:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784948977.078278:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948977.078281:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948977.078285:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:1.0:1784948977.078287:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000100:00000040:0.0:1784948977.078290:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2121 usecs (3 jiffies) 00000400:00000010:2.0:1784948977.078296:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:0.0:1784948977.078301:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305916421 00000400:00000200:3.0:1784948977.078303:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784948977.078314:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305916460 00000100:00000001:2.0:1784948977.078314:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784948977.078315:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289665792, offset 224 00000100:00000001:0.0:1784948977.078324:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305916421 00000400:00000200:3.0:1784948977.078327:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6439180 00000100:00000001:0.0:1784948977.078334:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305916460 00000100:00000200:2.0:1784948977.078334:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948977.078338:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784948977.078347:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305916421 00000100:00000001:0.0:1784948977.078356:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305916460 00000400:00000100:3.0:1784948977.078361:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289665920 offset 0 length 520: 4 00000400:00000200:1.0:1784948977.078367:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784948977.078368:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305916421 00000400:00000200:3.0:1784948977.078379:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000d20e62f1 is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:0.0:1784948977.078382:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305916460 00000400:00000200:3.0:1784948977.078390:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:0.0:1784948977.078392:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784948977.078397:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439100 00000100:00000001:0.0:1784948977.078405:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.078410:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:3.0:1784948977.078414:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948977.078415:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948977.078418:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8a31 [1] + 224 00000100:00000001:0.0:1784948977.078424:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948977.078426:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949002 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784948977.078428:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948977.078439:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784948977.078439:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948977.078447:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948977.078457:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784948977.078458:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784948977.078464:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948977.078474:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.078475:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948977.078476:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948977.078480:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.078487:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:1.0:1784948977.078488:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948977.078506:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784948977.078509:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948977.078536:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784948977.078547:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:1.0:1784948977.078550:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000100:00000001:1.0:1784948977.078556:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784948977.078560:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948977.078566:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948977.078569:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.078574:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948977.078578:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000800:00000200:3.0:1784948977.078584:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1784948977.078585:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:2.0:1784948977.078587:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:0.0:1784948977.078588:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948977.078592:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784948977.078595:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008746b1e6. 00000400:00000001:2.0:1784948977.078598:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784948977.078600:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:3.0:1784948977.078603:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948977.078606:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948977.078608:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d3f471e1. 00000100:00000040:1.0:1784948977.078611:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000019a655f1 x1871651289665792/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:638/0 lens 224/224 e 0 to 0 dl 1784948988 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948977.078617:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948977.078620:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948977.078626:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000800:00000001:0.0:1784948977.078627:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948977.078638:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000400:00000200:0.0:1784948977.078650:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948977.078653:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:3.0:1784948977.078658:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000100:00100000:1.0:1784948977.078662:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000019a655f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289665792:12345-0@lo:400: Request processed in 826us (1473us total) trans 0 rc 0/0 00000400:00000010:3.0:1784948977.078664:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000200:2.0:1784948977.078672:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.078677:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784948977.078685:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439200 00000100:00100000:1.0:1784948977.078689:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 748 00000100:00000200:3.0:1784948977.078692:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948977.078702:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784948977.078714:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000200:0.0:1784948977.078718:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 163216 00000100:00000001:1.0:1784948977.078721:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948977.078731:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004589cdd0. 00000020:00000010:1.0:1784948977.078745:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000800:00000001:0.0:1784948977.078749:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784948977.078752:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1784948977.078754:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dd0652fb. 00000100:00000200:2.0:1784948977.078757:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784948977.078764:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000040:3.0:1784948977.078767:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784948977.078774:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784948977.078776:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784948977.078784:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000057fac1f9. 00000400:00000200:0.0:1784948977.078787:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784948977.078799:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784948977.078808:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948977.078809:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:1.0:1784948977.078821:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784948977.078827:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784948977.078830:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:0.0:1784948977.078834:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784948977.078835:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948977.078836:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000026bb7060. 00000100:00000001:2.0:1784948977.078838:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1784948977.078847:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004a366adf. 00000100:00000001:3.0:1784948977.078849:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:1.0:1784948977.078851:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000016d64ec5. 00000100:00000001:2.0:1784948977.078852:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:0.0:1784948977.078858:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000004a366adf x1871651289666048 msgsize 224 02000000:00000001:2.0:1784948977.078863:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948977.078871:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000001:3.0:1784948977.078877:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948977.078878:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784948977.078880:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948977.078892:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948977.078894:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00001000:2.0:1784948977.078900:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=149 v=5 (5 5 5 5) 00000800:00000001:0.0:1784948977.078953:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:2.0:1784948977.078958:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=149 v=5 (1 1 1 1) 00000800:00000001:0.0:1784948977.078963:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948977.078968:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784948977.078970:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00100000:1.0:1784948977.078974:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666048 00000100:00000001:2.0:1784948977.078977:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948977.078980:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948977.078988:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00010000:00000001:2.0:1784948977.078990:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:1.0:1784948977.078991:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948977.079000:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948977.079001:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948977.079014:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666048 00000100:00000001:2.0:1784948977.079014:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948977.079020:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784948977.079025:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948977.079028:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784948977.079031:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:2.0:1784948977.079033:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948977.079039:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784948977.079048:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948977.079058:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784948977.079066:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:2.0:1784948977.079071:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948977.079073:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948977.079086:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:2.0:1784948977.079086:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948977.079095:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784948977.079097:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dd0652fb. 00000020:00000010:1.0:1784948977.079107:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004589cdd0. 00000020:00000010:1.0:1784948977.079113:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00000100:00000040:2.0:1784948977.079113:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948977.079126:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784948977.079134:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948977.079139:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948977.079171:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948977.079183:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00100000:2.0:1784948977.079185:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000df8ddd96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289665792:0@lo:400: 00000100:00000001:1.0:1784948977.079188:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784948977.079193:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.079206:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948977.079211:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948977.079213:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948977.079220:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000df8ddd96 x1871651289665792/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948977.079230:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948977.079236:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948977.079244:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1784948977.079251:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 724 02000000:00000001:2.0:1784948977.079254:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:1.0:1784948977.079257:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000010:2.0:1784948977.079263:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 00000100:00000001:1.0:1784948977.079264:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554977024 : -115544154574592 : ffff96e9c7b5f100) 00000100:00000040:1.0:1784948977.079279:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000004a366adf x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:638/0 lens 224/0 e 0 to 0 dl 1784948988 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948977.079280:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948977.079289:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948977.079296:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948977.079300:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:2.0:1784948977.079308:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000010:1.0:1784948977.079309:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 02000000:00000010:2.0:1784948977.079317:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a3cf4688. 00000100:00000001:1.0:1784948977.079318:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948977.079326:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 02000000:00000001:2.0:1784948977.079332:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:1.0:1784948977.079339:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000004a366adf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289666048:12345-192.168.202.137@tcp:400: 02000000:00000001:2.0:1784948977.079340:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948977.079348:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000df8ddd96. 00000100:00000200:1.0:1784948977.079350:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666048 00000020:00000001:1.0:1784948977.079359:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784948977.079362:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784948977.079364:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.079365:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1784948977.079368:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:2.0:1784948977.079372:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948977.079374:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784948977.079383:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948977.079384:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948977.079390:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948977.079395:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948977.079396:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784948977.079398:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948977.079400:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1784948977.079401:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948977.079408:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948977.079410:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784948977.079416:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948977.079420:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1784948977.079421:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000057fac1f9. 00000100:00000001:2.0:1784948977.079428:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948977.079429:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948977.079440:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948977.079443:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784948977.079446:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666048 00010000:00000001:1.0:1784948977.079449:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784948977.079453:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000004a366adf x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:638/0 lens 224/224 e 0 to 0 dl 1784948988 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948977.079463:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784948977.079466:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948977.079470:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=144 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948977.079474:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784948977.079478:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784948977.079481:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784948977.079483:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948977.079485:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948977.079488:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948977.079491:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784948977.079493:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000100:00000200:1.0:1784948977.079497:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289666048, offset 224 00000400:00000200:1.0:1784948977.079502:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948977.079511:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948977.079517:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3486:3486:238:4294967295] 192.168.202.137@tcp LPNI seq info [2242:2242:8:4294967295] 00000400:00000200:1.0:1784948977.079528:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948977.079534:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948977.079539:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000800:00000200:1.0:1784948977.079544:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948977.079551:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784948977.079554:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784948977.079571:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948977.079574:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784948977.079577:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784948977.079579:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948977.079581:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784948977.079585:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004a366adf x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:638/0 lens 224/224 e 0 to 0 dl 1784948988 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948977.079596:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000004a366adf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289666048:12345-192.168.202.137@tcp:400: Request processed in 295us (736us total) trans 0 rc 0/0 00000100:00100000:1.0:1784948977.079604:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 724 00000100:00000040:1.0:1784948977.079607:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784948977.079609:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948977.079612:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784948977.079615:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004589cdd0. 00000020:00000010:1.0:1784948977.079618:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000020:00000010:1.0:1784948977.079622:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dd0652fb. 00000020:00000040:1.0:1784948977.079625:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948977.079628:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948977.079746:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784948977.079753:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000400:00000200:3.0:1784948977.079760:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948977.079769:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784948977.079775:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:3.0:1784948977.079777:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000100:00000001:3.0:1784948977.079783:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784948977.079786:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784948977.079794:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948977.079806:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784948977.079810:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948977.079814:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948977.079824:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948977.079835:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439200 00000400:00000200:0.0:1784948977.079843:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8a41 [1] + 224 00000800:00000001:0.0:1784948977.079848:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948977.079853:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948977.079856:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948977.079860:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784948977.079865:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784948977.079870:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948977.079881:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948977.079904:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784948977.079942:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948977.079948:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.079951:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948977.079956:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948977.079963:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948977.079967:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948977.079970:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000968c2fa4. 00000400:00000200:3.0:1784948977.079977:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:3.0:1784948977.079978:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000100:00000001:3.0:1784948977.079981:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948977.079988:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948977.080001:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.080010:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948977.080012:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.080015:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948977.080017:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948977.080019:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948977.080021:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948977.080024:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948977.080030:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=144 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948977.080034:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=144 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948977.080036:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948977.080038:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948977.080041:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948977.080044:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948977.080062:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.080065:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948977.080068:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948977.080070:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948977.080071:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948977.080075:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.080082:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948977.080085:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948977.080087:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948977.080090:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948977.080098:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c1a2ac13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289666048:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948977.080104:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.080106:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948977.080108:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c1a2ac13 x1871651289666048/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948977.080117:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948977.080119:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948977.080121:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 02000000:00000001:3.0:1784948977.080124:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948977.080126:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948977.080128:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948977.080130:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948977.080132:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b84187a1. 02000000:00000001:3.0:1784948977.080134:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948977.080136:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948977.080138:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c1a2ac13. 00000100:00000001:3.0:1784948977.080173:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948977.080175:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.080177:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948977.080183:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948977.080185:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948977.080187:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948977.080190:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948977.120999:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305916460, at 4305916461 00000100:00000010:2.0:1784948977.121014:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a3d0cc2f. 00000100:00000040:3.0:1784948977.121026:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305916460, at 4305916461 00000100:00000010:3.0:1784948977.121041:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000100:00000040:0.0:1784948977.121047:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305916460, at 4305916461 00000100:00000040:1.0:1784948977.121055:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305916460, at 4305916461 00000100:00000010:0.0:1784948977.121061:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000010:1.0:1784948977.121070:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f4af3cce. 00010000:00000001:2.0:1784948977.908077:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948977.908089:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948977.908102:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948977.908105:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948977.908234:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948977.908240:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948977.908245:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948978.932048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948978.932059:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948978.932070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948978.932073:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948978.932215:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948978.932222:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948978.932226:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784948979.956056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948979.956065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948979.956074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948979.956076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948979.956219:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948979.956227:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948979.956233:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1784948980.660179:0:344056:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.9:988 00000400:00000100:0.0:1784948980.660191:0:344057:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.11:988 00000400:00000100:2.0:1784948980.660198:0:344054:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.5:988 00000400:00000100:3.0:1784948980.660208:0:344055:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.7:988 00000400:00000100:0.0:1784948980.660304:0:344057:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.11@tcp at host 111.222.142.11:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784948980.660310:0:344056:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.9@tcp at host 111.222.142.9:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784948980.660316:0:344055:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.7@tcp at host 111.222.142.7:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784948980.660326:0:344054:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.5@tcp at host 111.222.142.5:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784948980.660328:0:344057:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.11@tcp: down 00000400:00000200:1.0:1784948980.660338:0:344056:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.9@tcp: down 00000400:00000200:3.0:1784948980.660344:0:344055:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.7@tcp: down 00000400:00000200:0.0:1784948980.660345:0:344057:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948980.660358:0:344054:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.5@tcp: down 00000400:00000200:1.0:1784948980.660362:0:344056:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784948980.660372:0:344055:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784948980.660382:0:344056:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000400:00000100:0.0:1784948980.660385:0:344053:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.3:988 00000800:00000010:1.0:1784948980.660389:0:344056:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000005678171. 00000400:00000200:2.0:1784948980.660392:0:344054:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784948980.660404:0:344056:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:2.0:1784948980.660415:0:344054:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000400:00000200:1.0:1784948980.660420:0:344056:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000100:0.0:1784948980.660427:0:344053:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.3@tcp at host 111.222.142.3:988 took too long: that node may be hung or experiencing high load. 00000800:00000010:2.0:1784948980.660430:0:344054:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000400:00000200:1.0:1784948980.660440:0:344056:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11253 health :900 00000400:00000100:3.0:1784948980.660442:0:344052:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.1:988 00000400:00000200:2.0:1784948980.660445:0:344054:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948980.660446:0:344053:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.3@tcp: down 00000400:00000200:1.0:1784948980.660459:0:344056:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948980.660462:0:344053:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948980.660471:0:344054:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784948980.660475:0:344056:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:1.0:1784948980.660491:0:344056:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000100:3.0:1784948980.660504:0:344052:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.1@tcp at host 111.222.142.1:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784948980.660517:0:344054:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948980.660518:0:344052:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.1@tcp: down 00000400:00000200:3.0:1784948980.660530:0:344052:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784948980.660532:0:344054:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:0.0:1784948980.660543:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000100:3.0:1784948980.660544:0:344052:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000400:00000200:2.0:1784948980.660553:0:344054:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000800:00000010:3.0:1784948980.660555:0:344052:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000200:3.0:1784948980.660577:0:344052:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784948980.660583:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000200:3.0:1784948980.660592:0:344052:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000001:0.0:1784948980.660598:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948980.660603:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948980.660607:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000400:00000010:0.0:1784948980.660610:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000400:00000200:0.0:1784948980.660616:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784948980.660618:0:344052:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784948980.660625:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000200:3.0:1784948980.660630:0:344052:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000010:0.0:1784948980.660635:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004520da55. 00000400:00000200:0.0:1784948980.660645:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:3.0:1784948980.660649:0:344052:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:0.0:1784948980.660655:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:0.0:1784948980.660668:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:0.0:1784948980.660674:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:0.0:1784948980.660677:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948980.660679:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948980.660681:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 00000400:00000010:0.0:1784948980.660683:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000400:00000200:0.0:1784948980.660687:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948980.660690:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:0.0:1784948980.660693:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000059e5b358. 00000400:00000200:0.0:1784948980.660696:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:0.0:1784948980.660699:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:0.0:1784948980.660702:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000400:00000200:0.0:1784948980.660705:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:0.0:1784948980.660707:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948980.660708:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948980.660710:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000010:0.0:1784948980.660712:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000400:00000200:0.0:1784948980.660715:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948980.660717:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:0.0:1784948980.660719:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005cc3cd5d. 00000400:00000200:0.0:1784948980.660724:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000200:0.0:1784948980.660726:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:0.0:1784948980.660729:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00010000:00000001:2.0:1784948980.980000:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948980.980007:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948980.980013:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948980.980016:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948980.980018:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948980.980021:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948980.980023:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948980.980025:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948980.980027:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948980.980029:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948980.980147:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948980.980162:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948980.980165:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:3.0:1784948981.363977:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784948981.363986:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000041c2f022. 00000400:00000010:3.0:1784948981.363994:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006cd4526f. 00000400:00000010:3.0:1784948981.363999:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000400:00000010:3.0:1784948981.364005:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000009797c23. 00000400:00000200:3.0:1784948981.364008:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948981.364020:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948981.364025:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948981.364028:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3487:3487:241:4294967295] 192.168.202.137@tcp LPNI seq info [2243:2243:8:4294967295] 00000400:00000200:3.0:1784948981.364032:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784948981.364038:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784948981.364042:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948981.364045:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003547cd5f. 00000800:00000200:3.0:1784948981.364050:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948981.364057:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784948981.364060:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784948981.364074:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11255 health :700 00000400:00000010:3.0:1784948981.364079:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 0000000083930ef2. 00000400:02000040:3.0:1784948981.364081:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000800:00000001:1.0:1784948981.364247:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784948981.364248:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948981.364255:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003547cd5f. 00000800:00000001:1.0:1784948981.364259:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948981.364261:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948981.364264:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948981.364268:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784948981.364271:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948981.364274:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784948981.364283:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:1.0:1784948981.364283:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784948981.364303:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784948981.364311:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784948981.364318:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948981.364322:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3488:3488:241:4294967295] 192.168.202.137@tcp LPNI seq info [2244:2244:8:4294967295] 00000400:00000200:1.0:1784948981.364327:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784948981.364330:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784948981.364333:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c2b2d4f6. 00000800:00000200:1.0:1784948981.364336:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784948981.364340:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784948981.364342:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c2b2d4f6 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784948981.364386:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784948981.364389:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784948981.364391:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784948981.364392:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c2b2d4f6. 00000800:00000001:0.0:1784948981.364398:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948981.364400:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784948981.364402:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948981.364405:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784948981.364407:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948981.364411:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948981.364418:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784948981.364430:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8a55 00000800:00000001:0.0:1784948981.364435:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948981.364439:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948981.364442:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948981.364446:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784948981.364452:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:0.0:1784948981.364455:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000400:00000200:0.0:1784948981.364459:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784948981.364463:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000041c2f022. 00000400:00000010:0.0:1784948981.364467:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006cd4526f. 00000800:00000001:0.0:1784948981.364473:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784948981.364476:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948981.365982:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 0000000083930ef2. 00010000:00000001:2.0:1784948982.004020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948982.004030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948982.004038:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948982.004040:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948982.004187:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948982.004193:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948982.004198:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784948982.196049:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948982.196071:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948982.196081:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000047c147a0. 00000020:00000040:0.0:1784948982.196091:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948982.196097:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948982.196124:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948982.196132:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948982.196143:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000086e4d21. 00000100:00000001:3.0:1784948982.196146:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948982.196153:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.196161:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948982.196166:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948982.196169:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948982.196173:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948982.196180:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948982.196187:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948982.196196:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784948982.196197:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948982.196203:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948982.196213:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047c147a0] to pc [ptlrpcd_00_03+3] req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948982.196253:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948982.196257:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948982.196263:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948982.196266:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948982.196271:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948982.196275:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bb1b4c23. 00000020:00000040:0.0:1784948982.196278:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784948982.196282:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948982.196284:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948982.196288:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948982.196291:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d1b12746. 00000100:00000001:0.0:1784948982.196295:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.196299:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948982.196305:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1784948982.196310:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:1.0:1784948982.196319:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948982.196330:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948982.196333:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.196334:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948982.196340:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948982.196341:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb1b4c23] to pc [ptlrpcd_00_00+0] req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948982.196352:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948982.196355:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948982.196358:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948982.196363:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.196366:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784948982.196382:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948982.196391:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.196403:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784948982.196406:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11259) 00000100:00000001:3.0:1784948982.196407:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948982.196408:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.196416:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784948982.196420:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948982.196422:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784948982.196426:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784948982.196432:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948982.196435:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948982.196441:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000047c147a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289666176:0@lo:400: 00000100:00000001:0.0:1784948982.196466:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948982.196472:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948982.196475:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948982.196476:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948982.196481:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784948982.196482:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948982.196490:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948982.196492:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948982.196497:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1784948982.196498:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948982.196504:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948982.196507:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dd0652fb. 00000100:00100000:3.0:1784948982.196513:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000bb1b4c23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289666304:192.168.202.137@tcp:400: 02000000:00000001:1.0:1784948982.196515:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.196526:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:1.0:1784948982.196530:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000014bfbbb1. 02000000:00000001:3.0:1784948982.196532:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948982.196538:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948982.196542:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:1.0:1784948982.196544:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 02000000:00000010:3.0:1784948982.196549:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 02000000:00000001:3.0:1784948982.196555:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948982.196556:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666176, portal 25 00000100:00000001:1.0:1784948982.196562:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:3.0:1784948982.196564:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000100:00000001:1.0:1784948982.196566:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554976128 : -115544154575488 : ffff96e9c7b5ed80) 00000400:00000010:3.0:1784948982.196576:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000100:00000040:1.0:1784948982.196580:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948982.196585:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666304, portal 10 00000100:00000001:3.0:1784948982.196595:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948982.196598:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554979712 : -115544154571904 : ffff96e9c7b5fb80) 00000100:00000001:1.0:1784948982.196607:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948982.196616:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948982.196618:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948982.196628:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000100:00000001:3.0:1784948982.196637:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:1.0:1784948982.196639:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289666176, offset 0 00000100:00000040:3.0:1784948982.196649:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948982.196655:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784948982.196657:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000100:00000200:3.0:1784948982.196664:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289666304, offset 0 00000400:00000200:1.0:1784948982.196683:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784948982.196685:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948982.196708:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948982.196717:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784948982.196725:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948982.196731:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948982.196736:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948982.196741:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3489:3489:241:4294967295] 192.168.202.137@tcp LPNI seq info [2245:2245:8:4294967295] 00000400:00000200:3.0:1784948982.196759:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948982.196766:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784948982.196776:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439280 00000400:00000200:1.0:1784948982.196797:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8031 [2] + 0 00000400:00000010:1.0:1784948982.196803:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e3d6da39. 00000400:00000200:1.0:1784948982.196810:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000043dfa704 00000400:00000200:1.0:1784948982.196820:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948982.196826:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784948982.196834:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000043dfa704 00000400:00000010:1.0:1784948982.196837:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000043dfa704. 00000100:00000001:1.0:1784948982.196843:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784948982.196846:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784948982.196851:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000095b72665 x1871651289666176 msgsize 224 00000100:00100000:1.0:1784948982.196857:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784948982.196859:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784948982.196877:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000010:3.0:1784948982.196887:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000400:00000200:1.0:1784948982.196888:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784948982.196894:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000100:00000001:1.0:1784948982.196901:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784948982.196952:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948982.196955:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784948982.196966:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784948982.196977:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1784948982.196978:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666176 00000800:00000200:3.0:1784948982.196980:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:1.0:1784948982.196988:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948982.196991:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000200:3.0:1784948982.196995:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948982.197001:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948982.197011:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.197022:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948982.197025:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.197028:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784948982.197033:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948982.197040:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666176 00000100:00000001:1.0:1784948982.197041:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948982.197043:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2122 usecs (3 jiffies) 00000020:00000001:2.0:1784948982.197052:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948982.197055:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305921540 00000100:00000001:1.0:1784948982.197057:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305921579 00000020:00000040:2.0:1784948982.197062:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:3.0:1784948982.197069:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000020:00000001:2.0:1784948982.197072:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784948982.197081:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784948982.197084:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000001:2.0:1784948982.197094:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784948982.197133:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305921540 00000020:00000001:2.0:1784948982.197135:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784948982.197145:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000029366a5. 00000100:00000001:2.0:1784948982.197151:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948982.197155:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000010:2.0:1784948982.197163:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:3.0:1784948982.197167:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305921540 00000100:00000001:2.0:1784948982.197176:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1784948982.197180:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000020:00000010:2.0:1784948982.197189:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000001:3.0:1784948982.197190:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948982.197200:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305921540 00000100:00000010:3.0:1784948982.197210:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000006204bf6f. 00000020:00000010:2.0:1784948982.197211:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000100:00000001:3.0:1784948982.197217:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000010:2.0:1784948982.197226:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000057de5f1c. 00000100:00000001:3.0:1784948982.197229:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948982.197238:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948982.197242:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784948982.197244:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.197253:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948982.197261:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948982.197264:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948982.197269:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784948982.197270:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784948982.197273:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948982.197279:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948982.197283:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948982.197286:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948982.197289:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784948982.197290:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784948982.197300:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784948982.197301:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:0.0:1784948982.197314:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784948982.197324:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948982.197325:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948982.197339:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000400:00000010:0.0:1784948982.197342:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000100:00000001:0.0:1784948982.197348:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784948982.197355:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948982.197358:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948982.197365:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948982.197378:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784948982.197380:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305921540 00000800:00000001:3.0:1784948982.197385:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784948982.197387:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948982.197393:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784948982.197394:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305921579 00000800:00000001:3.0:1784948982.197395:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948982.197404:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:3.0:1784948982.197408:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948982.197409:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948982.197419:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784948982.197421:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 749 00000400:00000200:3.0:1784948982.197433:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:2.0:1784948982.197436:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948982.197445:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818128976 : -115538891422640 : ffff96eb016b4450) 00000400:00000200:3.0:1784948982.197455:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439300 00000100:00000001:1.0:1784948982.197461:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305921540 00000100:00000040:2.0:1784948982.197467:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000095b72665 x1871651289666176/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:643/0 lens 224/0 e 0 to 0 dl 1784948993 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948982.197474:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 163440 00000100:00000001:1.0:1784948982.197475:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305921579 00000100:00000001:1.0:1784948982.197492:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305921540 00000800:00000001:3.0:1784948982.197496:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948982.197511:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305921579 00000800:00000001:3.0:1784948982.197512:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948982.197514:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784948982.197525:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:2.0:1784948982.197528:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:1.0:1784948982.197529:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305921540 00000400:00000200:3.0:1784948982.197539:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948982.197544:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948982.197545:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948982.197550:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948982.197557:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784948982.197559:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784948982.197561:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000010:3.0:1784948982.197568:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a1a3e2fd. 00000100:00000001:1.0:1784948982.197569:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948982.197579:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000a1a3e2fd x1871651289666304 msgsize 224 00000100:00000001:1.0:1784948982.197581:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948982.197581:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000095b72665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289666176:12345-0@lo:400: 00000100:00000200:2.0:1784948982.197601:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666176 00000100:00100000:3.0:1784948982.197602:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:2.0:1784948982.197612:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784948982.197623:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784948982.197625:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:2.0:1784948982.197628:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948982.197637:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:3.0:1784948982.197639:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784948982.197642:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000001:3.0:1784948982.197648:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948982.197654:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948982.197661:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784948982.197667:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948982.197672:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784948982.197674:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:2.0:1784948982.197678:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948982.197680:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666304 00000020:00000001:2.0:1784948982.197689:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 02000000:00000001:1.0:1784948982.197690:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784948982.197698:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948982.197701:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784948982.197702:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948982.197712:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:1.0:1784948982.197713:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948982.197720:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666304 00000020:00000001:1.0:1784948982.197723:0:342764:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000010:2.0:1784948982.197724:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000052be9ded. 00000020:00000040:1.0:1784948982.197726:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784948982.197735:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1784948982.197736:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784948982.197740:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784948982.197745:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948982.197747:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948982.197757:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948982.197758:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784948982.197769:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000040:2.0:1784948982.197772:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666176 00000020:00000010:1.0:1784948982.197776:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00010000:00000001:2.0:1784948982.197781:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784948982.197783:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784948982.197790:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00010000:00000200:2.0:1784948982.197796:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000095b72665 x1871651289666176/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:643/0 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948982.197797:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000052e1e01a. 00000020:00000010:1.0:1784948982.197807:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008e869130. 00000020:00000010:1.0:1784948982.197815:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000003286e66a. 00010000:00000001:2.0:1784948982.197826:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:1.0:1784948982.197832:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00010000:00000001:2.0:1784948982.197833:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.197842:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00001000:2.0:1784948982.197843:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=154 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948982.197845:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948982.197852:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948982.197855:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784948982.197861:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948982.197862:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784948982.197865:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784948982.197868:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:2.0:1784948982.197872:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784948982.197873:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948982.197877:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948982.197884:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784948982.197887:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948982.197893:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:1.0:1784948982.197897:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784948982.197899:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289666176, offset 224 00000400:00000200:2.0:1784948982.197958:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784948982.197963:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948982.197963:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784948982.197971:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784948982.197972:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784948982.197975:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:3.0:1784948982.197985:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 725 00000400:00000200:2.0:1784948982.197993:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439280 00000100:00000040:3.0:1784948982.197997:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948982.197998:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.198007:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554955264 : -115544154596352 : ffff96e9c7b59c00) 00000400:00000200:2.0:1784948982.198008:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8a59 [1] + 224 00000100:00000001:1.0:1784948982.198010:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948982.198024:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a1a3e2fd x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:643/0 lens 224/0 e 0 to 0 dl 1784948993 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948982.198028:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948982.198045:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784948982.198052:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948982.198055:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784948982.198061:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000200:2.0:1784948982.198063:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948982.198070:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948982.198082:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000100:00000040:2.0:1784948982.198098:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948982.198123:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a1a3e2fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289666304:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784948982.198141:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666304 00000020:00000001:3.0:1784948982.198148:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784948982.198155:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784948982.198159:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948982.198163:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948982.198166:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784948982.198169:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1784948982.198170:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784948982.198176:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948982.198180:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:2.0:1784948982.198181:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000020:00000001:3.0:1784948982.198185:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:2.0:1784948982.198187:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000020:00000001:3.0:1784948982.198190:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948982.198194:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:3.0:1784948982.198196:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948982.198199:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784948982.198204:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948982.198208:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948982.198211:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784948982.198219:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 02000000:00000010:3.0:1784948982.198219:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000011ff6ec2. 00000100:00000001:1.0:1784948982.198229:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948982.198230:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948982.198231:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948982.198239:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.198240:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948982.198241:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948982.198245:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:3.0:1784948982.198251:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948982.198257:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000095b72665 x1871651289666176/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:643/0 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784948982.198258:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000040:3.0:1784948982.198259:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666304 00000400:00000001:1.0:1784948982.198267:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:3.0:1784948982.198271:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000010:1.0:1784948982.198277:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000014bfbbb1. 00010000:00000200:3.0:1784948982.198284:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a1a3e2fd x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:643/0 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948982.198291:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000095b72665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289666176:12345-0@lo:400: Request processed in 770us (1431us total) trans 0 rc 0/0 00000400:00000200:1.0:1784948982.198293:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:1.0:1784948982.198305:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000100:00000001:1.0:1784948982.198319:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784948982.198324:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 749 00010000:00000001:3.0:1784948982.198325:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948982.198332:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784948982.198334:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948982.198335:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00001000:3.0:1784948982.198337:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=149 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948982.198345:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948982.198353:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948982.198354:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1784948982.198363:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000057de5f1c. 00000100:00000040:3.0:1784948982.198366:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948982.198376:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000010:2.0:1784948982.198377:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000200:1.0:1784948982.198382:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948982.198384:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000010:2.0:1784948982.198388:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 02000000:00000001:3.0:1784948982.198389:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.198401:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1784948982.198405:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784948982.198410:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1784948982.198415:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948982.198416:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784948982.198418:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784948982.198425:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000052be9ded. 00000400:00000010:3.0:1784948982.198426:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000001:1.0:1784948982.198427:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948982.198437:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:1.0:1784948982.198438:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000200:3.0:1784948982.198438:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289666304, offset 224 00000100:00000001:2.0:1784948982.198443:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948982.198445:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:3.0:1784948982.198454:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784948982.198455:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948982.198456:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:1.0:1784948982.198464:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948982.198469:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d3f471e1. 00000400:00000200:3.0:1784948982.198472:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00001000:1.0:1784948982.198474:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=154 v=5 (5 5 5 5) 00000400:00000010:2.0:1784948982.198479:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a1b701f2. 00000100:00001000:1.0:1784948982.198492:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=154 v=5 (1 1 1 1) 00000400:00000200:3.0:1784948982.198493:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3490:3490:241:4294967295] 192.168.202.137@tcp LPNI seq info [2246:2246:8:4294967295] 00000100:00000001:1.0:1784948982.198507:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948982.198511:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948982.198520:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784948982.198526:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948982.198528:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784948982.198531:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948982.198540:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784948982.198542:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1784948982.198551:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000100:00000001:1.0:1784948982.198564:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1784948982.198565:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784948982.198574:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948982.198580:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784948982.198580:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:1.0:1784948982.198590:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784948982.198591:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00100000:1.0:1784948982.198610:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000047c147a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289666176:0@lo:400: 00000100:00000001:1.0:1784948982.198616:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948982.198619:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948982.198620:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948982.198627:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000047c147a0 x1871651289666176/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948982.198631:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948982.198640:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948982.198645:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948982.198652:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784948982.198658:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:3.0:1784948982.198664:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a1a3e2fd x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:643/0 lens 224/224 e 0 to 0 dl 1784948993 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948982.198666:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948982.198671:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dd0652fb. 02000000:00000001:1.0:1784948982.198681:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948982.198688:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948982.198694:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00100000:3.0:1784948982.198695:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a1a3e2fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289666304:12345-192.168.202.137@tcp:400: Request processed in 636us (1107us total) trans 0 rc 0/0 00000020:00000001:1.0:1784948982.198705:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948982.198712:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000086e4d21. 00000100:00100000:3.0:1784948982.198718:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 725 02000000:00000001:1.0:1784948982.198719:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:3.0:1784948982.198727:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 02000000:00000001:1.0:1784948982.198728:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948982.198734:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000047c147a0. 00000100:00000001:3.0:1784948982.198735:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948982.198743:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948982.198748:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008e869130. 00000020:00000010:3.0:1784948982.198755:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000003286e66a. 00000100:00000001:1.0:1784948982.198759:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000010:3.0:1784948982.198761:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000052e1e01a. 00000100:00000001:1.0:1784948982.198764:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948982.198769:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784948982.198773:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784948982.198776:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948982.198781:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.198781:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948982.198790:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948982.198795:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784948982.198835:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784948982.198842:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:0.0:1784948982.198850:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784948982.198852:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948982.198860:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948982.198867:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000800:00000001:1.0:1784948982.198868:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948982.198871:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000800:00000001:1.0:1784948982.198875:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948982.198878:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784948982.198881:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948982.198883:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784948982.198896:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784948982.198941:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439300 00000400:00000200:1.0:1784948982.198949:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8a5d [1] + 224 00000800:00000001:1.0:1784948982.198956:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948982.198965:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948982.198968:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784948982.198972:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784948982.198977:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948982.198981:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948982.198992:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948982.199014:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784948982.199021:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948982.199024:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948982.199063:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948982.199067:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948982.199072:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948982.199074:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948982.199077:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000400:00000200:3.0:1784948982.199084:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000400:00000010:3.0:1784948982.199086:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000100:00000001:3.0:1784948982.199090:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948982.199096:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948982.199131:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948982.199141:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948982.199144:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948982.199147:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948982.199150:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948982.199151:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948982.199154:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948982.199156:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948982.199162:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=149 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948982.199166:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=149 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948982.199169:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948982.199171:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948982.199173:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948982.199177:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948982.199185:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.199189:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948982.199190:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948982.199192:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948982.199194:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948982.199197:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948982.199205:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948982.199208:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948982.199210:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948982.199213:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948982.199222:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000bb1b4c23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289666304:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948982.199226:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948982.199229:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948982.199231:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000bb1b4c23 x1871651289666304/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948982.199238:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948982.199240:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948982.199243:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 02000000:00000001:3.0:1784948982.199248:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948982.199250:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948982.199251:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948982.199254:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948982.199256:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d1b12746. 02000000:00000001:3.0:1784948982.199259:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948982.199261:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948982.199262:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000bb1b4c23. 00000100:00000001:3.0:1784948982.199269:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948982.199271:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948982.199273:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948982.199277:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948982.199279:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948982.199281:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948982.199284:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948982.240033:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305921579, at 4305921580 00000100:00000040:0.0:1784948982.240041:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305921579, at 4305921580 00000100:00000040:1.0:1784948982.240054:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305921579, at 4305921580 00000100:00000010:2.0:1784948982.240056:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00000100:00000040:3.0:1784948982.240057:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305921579, at 4305921580 00000100:00000010:0.0:1784948982.240059:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000100:00000010:1.0:1784948982.240074:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000029366a5. 00000100:00000010:3.0:1784948982.240078:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000006204bf6f. 00010000:00000001:2.0:1784948983.028007:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948983.028015:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948983.028021:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948983.028023:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948983.028133:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948983.028139:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948983.028143:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948983.220033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948983.220049:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948983.220052:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948983.220058:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948983.220062:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948983.220066:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948983.220069:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948984.052038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948984.052051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948984.052063:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948984.052066:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948984.052143:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948984.052149:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948984.052154:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948984.244104:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948984.244123:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948984.244125:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948984.244131:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948984.244134:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948984.244138:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948984.244141:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948985.076041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948985.076050:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948985.076058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948985.076060:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948985.076182:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948985.076189:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948985.076193:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948985.268017:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948985.268030:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948985.268032:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948985.268037:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948985.268040:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948985.268045:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948985.268047:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948986.100037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948986.100046:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948986.100055:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948986.100057:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948986.100190:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948986.100197:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948986.100202:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948986.292169:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948986.292185:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948986.292189:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948986.292194:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948986.292199:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948986.292203:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948986.292208:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948987.124011:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948987.124021:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948987.124028:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948987.124030:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948987.124153:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948987.124160:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948987.124164:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784948987.316121:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948987.316132:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948987.316141:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a612e3ff. 00000100:00000001:3.0:1784948987.316144:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000020:00000040:1.0:1784948987.316149:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784948987.316158:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948987.316159:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1784948987.316162:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948987.316164:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948987.316172:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784948987.316173:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784948987.316185:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:1.0:1784948987.316194:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948987.316206:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784948987.316222:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784948987.316225:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948987.316232:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a612e3ff] to pc [ptlrpcd_00_01+1] req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948987.316275:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784948987.316279:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784948987.316284:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784948987.316289:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784948987.316294:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784948987.316298:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000069457997. 00000020:00000040:1.0:1784948987.316302:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784948987.316305:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784948987.316307:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784948987.316309:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.316313:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1784948987.316315:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000001:2.0:1784948987.316323:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948987.316326:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948987.316326:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948987.316329:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784948987.316334:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948987.316340:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948987.316345:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784948987.316347:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784948987.316362:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948987.316369:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.316401:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784948987.316408:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948987.316413:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784948987.316417:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784948987.316418:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948987.316424:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784948987.316430:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069457997] to pc [ptlrpcd_00_02+2] req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784948987.316432:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948987.316447:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289666432:0@lo:400: 00000100:00000001:2.0:1784948987.316457:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784948987.316464:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948987.316467:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948987.316471:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784948987.316478:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948987.316478:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007a6587e5. 02000000:00000001:2.0:1784948987.316483:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948987.316489:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11264) 00000400:00000010:2.0:1784948987.316494:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000603fb577. 00000400:00000010:2.0:1784948987.316508:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:1.0:1784948987.316512:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784948987.316519:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666432, portal 25 00000100:00000001:3.0:1784948987.316519:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784948987.316521:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:3.0:1784948987.316527:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948987.316532:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948987.316532:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948987.316538:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549158016 : -115544160393600 : ffff96e9c75d2680) 00000100:00000001:1.0:1784948987.316545:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1784948987.316550:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948987.316552:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.316561:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948987.316565:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784948987.316570:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948987.316576:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948987.316583:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948987.316587:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784948987.316595:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000100:00000001:1.0:1784948987.316602:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:2.0:1784948987.316603:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289666432, offset 0 00000100:00000001:1.0:1784948987.316608:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948987.316614:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784948987.316619:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784948987.316620:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784948987.316627:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784948987.316633:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289666560:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948987.316639:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784948987.316644:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1784948987.316648:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:1.0:1784948987.316650:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948987.316658:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784948987.316665:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dd0652fb. 02000000:00000001:1.0:1784948987.316668:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948987.316676:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439380 00000400:00000010:1.0:1784948987.316680:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000014bfbbb1. 00000400:00000200:2.0:1784948987.316690:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x804d [2] + 0 00000400:00000010:2.0:1784948987.316696:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f301fed7. 00000400:00000200:2.0:1784948987.316702:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000be708ecf 00000400:00000010:1.0:1784948987.316711:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000200:2.0:1784948987.316712:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1784948987.316722:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666560, portal 10 00000400:00000200:2.0:1784948987.316723:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784948987.316731:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784948987.316737:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549155328 : -115544160396288 : ffff96e9c75d1c00) 00000400:00000200:2.0:1784948987.316737:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000be708ecf 00000100:00000040:1.0:1784948987.316750:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948987.316761:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000be708ecf. 00000100:00000001:1.0:1784948987.316763:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784948987.316775:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784948987.316776:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784948987.316781:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:1.0:1784948987.316783:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000100:00000040:2.0:1784948987.316788:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f911f57b x1871651289666432 msgsize 224 00000100:00000200:1.0:1784948987.316790:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289666560, offset 0 00000400:00000200:1.0:1784948987.316808:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:2.0:1784948987.316809:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784948987.316817:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784948987.316828:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784948987.316836:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784948987.316837:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784948987.316845:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784948987.316852:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784948987.316857:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784948987.316863:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3491:3491:241:4294967295] 192.168.202.137@tcp LPNI seq info [2247:2247:8:4294967295] 00000400:00000200:1.0:1784948987.316879:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784948987.316890:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948987.316893:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000100:00000001:0.0:1784948987.316899:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000010:1.0:1784948987.316900:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000031deb17e. 00000400:00000010:2.0:1784948987.316900:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000100:00100000:0.0:1784948987.316938:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666432 02000000:00000001:0.0:1784948987.316943:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784948987.316951:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948987.316952:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784948987.316956:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784948987.316962:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:2.0:1784948987.316965:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948987.316967:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948987.316978:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666432 00000800:00000200:1.0:1784948987.316983:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:0.0:1784948987.316986:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784948987.316994:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000200:1.0:1784948987.316996:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000031deb17e type 1, nob 320 niov 1 nkiov 1 00000020:00000001:0.0:1784948987.317002:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784948987.317011:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:0.0:1784948987.317012:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784948987.317020:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:2.0:1784948987.317021:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948987.317025:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948987.317030:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:1.0:1784948987.317040:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2123 usecs (3 jiffies) 00000100:00000001:0.0:1784948987.317042:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948987.317048:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948987.317050:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305926660 00000020:00000010:0.0:1784948987.317052:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:2.0:1784948987.317056:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:1.0:1784948987.317082:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:0.0:1784948987.317086:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948987.317090:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948987.317094:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000100:00000001:1.0:1784948987.317095:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784948987.317099:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305926699 00000020:00000010:0.0:1784948987.317111:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001879354a. 00000100:00000001:1.0:1784948987.317112:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305926660 00000100:00000001:2.0:1784948987.317116:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305926660 00000100:00000010:1.0:1784948987.317129:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000010:0.0:1784948987.317130:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000093f05992. 00000100:00000001:2.0:1784948987.317132:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305926699 00000100:00000001:1.0:1784948987.317141:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000010:0.0:1784948987.317144:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784948987.317153:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305926660 00000100:00000001:2.0:1784948987.317160:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305926660 00000100:00000010:1.0:1784948987.317163:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000040:0.0:1784948987.317164:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948987.317173:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305926699 00000100:00000001:0.0:1784948987.317175:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948987.317176:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784948987.317183:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784948987.317188:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948987.317189:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305926660 00000100:00000001:2.0:1784948987.317196:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305926660 00000100:00000010:1.0:1784948987.317198:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:0.0:1784948987.317201:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948987.317210:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784948987.317212:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305926699 00000100:00000001:1.0:1784948987.317221:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948987.317224:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.317227:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.317228:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305926660 00000100:00000001:1.0:1784948987.317237:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.317243:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948987.317245:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.317251:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948987.317253:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948987.317261:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948987.317262:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.317268:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784948987.317273:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 750 00000100:00000040:0.0:1784948987.317280:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784948987.317281:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948987.317288:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948987.317289:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818136144 : -115538891415472 : ffff96eb016b6050) 00000100:00000001:2.0:1784948987.317299:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784948987.317305:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f911f57b x1871651289666432/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:648/0 lens 224/0 e 0 to 0 dl 1784948998 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948987.317306:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784948987.317314:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.317323:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948987.317334:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948987.317341:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:0.0:1784948987.317346:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948987.317350:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000800:00000200:3.0:1784948987.317356:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00100000:0.0:1784948987.317359:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f911f57b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289666432:12345-0@lo:400: 00000800:00000010:3.0:1784948987.317365:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000031deb17e. 00000100:00000200:0.0:1784948987.317369:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666432 00000020:00000001:0.0:1784948987.317377:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:3.0:1784948987.317378:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784948987.317379:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784948987.317386:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948987.317390:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948987.317396:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784948987.317398:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948987.317399:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948987.317406:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000001:2.0:1784948987.317408:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784948987.317409:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000020:00000001:0.0:1784948987.317415:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:3.0:1784948987.317417:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000020:00000001:0.0:1784948987.317420:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948987.317421:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948987.317427:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784948987.317431:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948987.317437:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:3.0:1784948987.317442:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784948987.317445:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948987.317448:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:0.0:1784948987.317456:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948987.317463:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948987.317469:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784948987.317474:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784948987.317475:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439400 02000000:00000010:0.0:1784948987.317478:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b91d4ef5. 00000100:00000040:3.0:1784948987.317486:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784948987.317489:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948987.317497:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 163664 00000100:00000001:0.0:1784948987.317499:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948987.317509:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948987.317518:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000040:0.0:1784948987.317519:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666432 00010000:00000001:0.0:1784948987.317531:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:2.0:1784948987.317532:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948987.317533:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784948987.317545:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948987.317545:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000200:0.0:1784948987.317546:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f911f57b x1871651289666432/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:648/0 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784948987.317565:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000001:0.0:1784948987.317570:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948987.317574:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.317576:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00001000:0.0:1784948987.317579:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=9 v=5 (1 1 1 1) 00000100:00000200:2.0:1784948987.317581:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784948987.317595:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1784948987.317596:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c39699a8. 00000100:00000040:0.0:1784948987.317603:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:2.0:1784948987.317607:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000c39699a8 x1871651289666560 msgsize 224 00000100:00000001:0.0:1784948987.317609:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948987.317617:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784948987.317623:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784948987.317630:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784948987.317631:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784948987.317641:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784948987.317648:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:0.0:1784948987.317649:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00000100:00000200:0.0:1784948987.317658:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289666432, offset 224 00000800:00000001:2.0:1784948987.317664:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784948987.317667:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:2.0:1784948987.317674:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948987.317688:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784948987.317699:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784948987.317706:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666560 00000400:00000200:0.0:1784948987.317708:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439380 02000000:00000001:3.0:1784948987.317711:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948987.317717:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948987.317721:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948987.317723:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8a75 [1] + 224 02000000:00000001:3.0:1784948987.317731:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784948987.317736:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:3.0:1784948987.317737:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666560 00000020:00000001:3.0:1784948987.317746:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948987.317751:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:0.0:1784948987.317754:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:3.0:1784948987.317757:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784948987.317762:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:3.0:1784948987.317764:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784948987.317771:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:0.0:1784948987.317774:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948987.317777:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784948987.317789:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948987.317797:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000094fcdc7f. 00000100:00000040:0.0:1784948987.317801:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948987.317804:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784948987.317815:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000094fcdc7f. 00000020:00000010:3.0:1784948987.317827:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006cd4526f. 00000020:00000010:3.0:1784948987.317835:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000d6164f7. 00000020:00000010:3.0:1784948987.317840:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784948987.317845:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1784948987.317853:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:0.0:1784948987.317859:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000100:00000001:3.0:1784948987.317860:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784948987.317866:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000010:0.0:1784948987.317867:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000100:00000001:3.0:1784948987.317873:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948987.317875:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948987.317880:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784948987.317883:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784948987.317888:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948987.317890:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784948987.317896:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948987.317899:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784948987.317901:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948987.317929:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948987.317936:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000001:0.0:1784948987.317942:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:2.0:1784948987.317943:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784948987.317944:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948987.317949:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784948987.317950:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000603fb577. 00000020:00000001:0.0:1784948987.317956:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:2.0:1784948987.317962:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000100:00000001:3.0:1784948987.317968:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784948987.317969:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f911f57b x1871651289666432/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:648/0 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948987.317970:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00000001:3.0:1784948987.317975:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948987.317983:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784948987.317991:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784948987.317993:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 726 00000100:00000200:2.0:1784948987.317998:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948987.317999:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784948987.318002:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f911f57b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289666432:12345-0@lo:400: Request processed in 660us (1206us total) trans 0 rc 0/0 00000100:00000040:3.0:1784948987.318009:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784948987.318010:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948987.318021:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558302976 : -115544151248640 : ffff96e9c7e8b100) 00000100:00000001:1.0:1784948987.318022:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948987.318037:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 750 00000100:00000040:3.0:1784948987.318040:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000c39699a8 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:648/0 lens 224/0 e 0 to 0 dl 1784948998 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948987.318050:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784948987.318084:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:2.0:1784948987.318091:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948987.318092:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784948987.318095:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784948987.318101:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000093f05992. 00000020:00000010:3.0:1784948987.318104:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000094fcdc7f. 00000100:00000001:3.0:1784948987.318114:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948987.318115:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784948987.318121:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784948987.318127:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000094fcdc7f. 00000100:00000001:2.0:1784948987.318129:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948987.318130:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001879354a. 00000100:00000001:2.0:1784948987.318140:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00100000:3.0:1784948987.318142:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000c39699a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289666560:12345-192.168.202.137@tcp:400: 00000020:00000040:0.0:1784948987.318144:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:2.0:1784948987.318145:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948987.318152:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784948987.318156:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784948987.318158:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784948987.318159:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666560 02000000:00000010:0.0:1784948987.318162:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000b91d4ef5. 00000020:00000001:3.0:1784948987.318168:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:2.0:1784948987.318169:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948987.318171:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:3.0:1784948987.318176:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784948987.318180:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948987.318181:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948987.318182:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=9 v=5 (5 5 5 5) 00000100:00000200:0.0:1784948987.318188:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:3.0:1784948987.318191:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00001000:2.0:1784948987.318195:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=9 v=5 (1 1 1 1) 00000020:00000001:3.0:1784948987.318198:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000010:0.0:1784948987.318201:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007a72fba7. 00000100:00000001:2.0:1784948987.318209:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784948987.318211:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:0.0:1784948987.318213:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005df8d94f. 00000100:00000001:2.0:1784948987.318215:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948987.318219:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948987.318225:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784948987.318226:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00010000:00000001:2.0:1784948987.318231:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948987.318232:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948987.318238:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.318240:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948987.318244:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:2.0:1784948987.318252:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948987.318252:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784948987.318259:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784948987.318268:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d331fbd8. 00000100:00000001:2.0:1784948987.318269:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948987.318276:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.318277:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948987.318285:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.318286:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948987.318296:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948987.318299:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:3.0:1784948987.318302:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666560 00010000:00000001:3.0:1784948987.318312:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948987.318323:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000c39699a8 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:648/0 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948987.318327:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a612e3ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289666432:0@lo:400: 00000100:00000001:2.0:1784948987.318343:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948987.318354:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:3.0:1784948987.318354:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948987.318361:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948987.318365:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a612e3ff x1871651289666432/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:3.0:1784948987.318368:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=154 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948987.318382:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784948987.318391:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:3.0:1784948987.318391:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948987.318397:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784948987.318399:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1784948987.318404:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000010:2.0:1784948987.318405:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007a6587e5. 02000000:00000001:3.0:1784948987.318409:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948987.318414:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948987.318417:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784948987.318418:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784948987.318424:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000010:3.0:1784948987.318426:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000020:00000040:2.0:1784948987.318428:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784948987.318437:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000200:3.0:1784948987.318438:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289666560, offset 224 02000000:00000010:2.0:1784948987.318443:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 00000400:00000200:3.0:1784948987.318451:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784948987.318453:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948987.318457:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948987.318459:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a612e3ff. 00000400:00000200:3.0:1784948987.318466:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784948987.318467:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948987.318473:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948987.318478:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948987.318482:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3492:3492:241:4294967295] 192.168.202.137@tcp LPNI seq info [2248:2248:8:4294967295] 00000100:00000001:2.0:1784948987.318486:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948987.318491:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948987.318499:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948987.318505:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948987.318509:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948987.318518:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948987.318523:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784948987.318531:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948987.318538:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948987.318542:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948987.318560:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948987.318563:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948987.318566:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948987.318568:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948987.318571:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948987.318575:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c39699a8 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:648/0 lens 224/224 e 0 to 0 dl 1784948998 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948987.318586:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000c39699a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289666560:12345-192.168.202.137@tcp:400: Request processed in 491us (979us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948987.318595:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 726 00000100:00000040:3.0:1784948987.318598:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948987.318601:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948987.318603:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948987.318607:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000d6164f7. 00000020:00000010:3.0:1784948987.318611:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784948987.318615:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006cd4526f. 00000020:00000040:3.0:1784948987.318618:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948987.318621:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784948987.318777:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784948987.318785:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:1.0:1784948987.318792:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948987.318800:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784948987.318801:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784948987.318812:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000400:00000010:1.0:1784948987.318815:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000800:00000001:3.0:1784948987.318816:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.318820:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784948987.318823:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948987.318825:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784948987.318830:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948987.318841:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784948987.318857:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439400 00000400:00000200:3.0:1784948987.318864:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8a7d [1] + 224 00000800:00000001:3.0:1784948987.318872:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948987.318882:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948987.318885:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948987.318890:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948987.318896:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948987.318900:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948987.318939:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948987.318959:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948987.318966:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948987.318970:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.319010:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948987.319015:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784948987.319020:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784948987.319023:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784948987.319025:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000014bfbbb1. 00000400:00000200:1.0:1784948987.319030:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:1.0:1784948987.319032:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000100:00000001:1.0:1784948987.319035:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784948987.319041:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784948987.319054:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948987.319088:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784948987.319090:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.319094:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784948987.319097:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784948987.319099:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948987.319101:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784948987.319104:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784948987.319110:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=154 v=5 (5 5 5 5) 00000100:00001000:1.0:1784948987.319114:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=154 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948987.319117:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784948987.319119:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948987.319121:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784948987.319125:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784948987.319135:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948987.319138:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784948987.319140:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784948987.319143:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784948987.319144:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948987.319148:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948987.319155:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784948987.319158:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784948987.319160:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784948987.319163:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948987.319171:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000069457997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289666560:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784948987.319176:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.319179:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784948987.319181:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000069457997 x1871651289666560/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948987.319189:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784948987.319191:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784948987.319194:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dd0652fb. 02000000:00000001:1.0:1784948987.319198:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784948987.319200:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784948987.319202:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784948987.319205:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784948987.319207:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 02000000:00000001:1.0:1784948987.319210:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784948987.319212:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784948987.319213:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000069457997. 00000100:00000001:1.0:1784948987.319217:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784948987.319218:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.319221:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948987.319225:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784948987.319227:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948987.319230:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784948987.319232:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948987.360035:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305926699, at 4305926700 00000100:00000010:0.0:1784948987.360050:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:3.0:1784948987.360055:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305926699, at 4305926700 00000100:00000040:1.0:1784948987.360091:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305926699, at 4305926700 00000100:00000010:3.0:1784948987.360094:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:2.0:1784948987.360096:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305926699, at 4305926700 00000100:00000010:1.0:1784948987.360109:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:2.0:1784948987.360113:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784948988.148011:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948988.148019:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948988.148025:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948988.148027:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948988.148148:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948988.148154:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948988.148159:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948988.340030:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948988.340045:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948988.340048:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948988.340075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948988.340078:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948988.340082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948988.340085:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948989.172040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948989.172099:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948989.172108:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948989.172110:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948989.172228:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948989.172235:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948989.172239:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948989.364038:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948989.364086:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948989.364088:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948989.364093:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948989.364097:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948989.364101:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948989.364104:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784948989.775591:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948989.775631:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948989.775640:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948989.775643:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784948990.196096:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948990.196106:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948990.196114:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948990.196116:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948990.196209:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948990.196216:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948990.196220:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784948990.387998:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784948990.388012:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948990.388015:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948990.388019:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948990.388023:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948990.388026:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948990.388029:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948991.220229:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948991.220238:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948991.220245:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948991.220248:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784948991.220251:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784948991.220254:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948991.220257:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784948991.220260:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948991.220262:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948991.220264:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948991.220343:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948991.220350:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948991.220355:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948991.412128:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948991.412145:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948991.412147:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948991.412152:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948991.412156:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948991.412159:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948991.412162:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948991.604065:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000da72cb2d. 00000400:00000010:3.0:1784948991.604096:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000991db8b9. 00000400:00000010:3.0:1784948991.604104:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000098d04a76. 00000400:00000010:3.0:1784948991.604108:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c521f738. 00000400:00000010:3.0:1784948991.604113:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000dc3e31db. 00000400:00000010:3.0:1784948991.604116:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003e1fe5c5. 00000400:00000010:3.0:1784948991.604121:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000028b23cff. 00000400:00000010:3.0:1784948991.604124:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000284efd3d. 00000400:00000010:3.0:1784948991.604128:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004bff3bbc. 00000400:00000010:3.0:1784948991.604131:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000573e3d89. 00000400:00000010:3.0:1784948991.604135:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000078e6b4ca. 00000400:00000010:3.0:1784948991.604139:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000006233a0b. 00000400:00000010:3.0:1784948991.604143:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000005f31e551. 00000400:00000010:3.0:1784948991.604147:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c92a48c0. 00000400:00000010:3.0:1784948991.604150:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003dae5c32. 00000400:00000010:3.0:1784948991.604154:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001b14ffb3. 00000400:00000010:3.0:1784948991.604158:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000fce68683. 00000400:00000010:3.0:1784948991.604162:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000090e76bb9. 00000400:00000010:3.0:1784948991.604166:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ead52bd9. 00000400:00000010:3.0:1784948991.604169:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001da9c7e9. 00000400:00000010:3.0:1784948991.604174:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a1e94473. 00000400:00000010:3.0:1784948991.604177:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000816e8b2e. 00000400:00000010:3.0:1784948991.604194:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000055bdc3fa. 00000400:00000010:3.0:1784948991.604199:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a0ec5307. 00000400:00000010:3.0:1784948991.604203:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000099ed7e50. 00000400:00000010:3.0:1784948991.604207:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000168b2dd4. 00000400:00000010:3.0:1784948991.604211:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bde5615f. 00000400:00000010:3.0:1784948991.604214:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000007473943. 00000400:00000010:3.0:1784948991.604218:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ce76e0d8. 00000400:00000010:3.0:1784948991.604222:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000034b6aef4. 00000400:00000010:3.0:1784948991.604227:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002a00e572. 00000400:00000010:3.0:1784948991.604232:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000068539c1f. 00000400:00000010:3.0:1784948991.604236:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b76552f6. 00000400:00000010:3.0:1784948991.604240:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001fa0e430. 00000400:00000010:3.0:1784948991.604246:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002e0c2c48. 00000400:00000010:3.0:1784948991.604250:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e360eb6a. 00000400:00000010:3.0:1784948991.604255:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003ad675ec. 00000400:00000010:3.0:1784948991.604259:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008e22c077. 00000400:00000010:3.0:1784948991.604264:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000033c8520. 00000400:00000010:3.0:1784948991.604268:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bd6a7269. 00000400:00000010:3.0:1784948991.604272:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001fcbecf4. 00000400:00000010:3.0:1784948991.604277:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000061508776. 00000400:00000010:3.0:1784948991.604282:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000570fe348. 00000400:00000010:3.0:1784948991.604287:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000005243e367. 00000400:00000010:3.0:1784948991.604291:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000092c40cf9. 00000400:00000010:3.0:1784948991.604294:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000076669228. 00000400:00000010:3.0:1784948991.604300:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000feb070e9. 00010000:00000001:2.0:1784948992.244083:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948992.244093:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948992.244101:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948992.244102:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948992.244198:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948992.244204:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948992.244208:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948992.435999:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948992.436014:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948992.436030:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948992.436036:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948992.436039:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948992.436042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948992.436045:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948992.436090:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948992.436096:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948992.436105:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000020:00000040:2.0:1784948992.436112:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784948992.436117:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948992.436121:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784948992.436126:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784948992.436134:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000093457b1b. 00000100:00000001:2.0:1784948992.436141:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948992.436151:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784948992.436162:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948992.436164:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948992.436169:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3c18d2b] to pc [ptlrpcd_00_03+3] req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948992.436205:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784948992.436208:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784948992.436212:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784948992.436215:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784948992.436218:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784948992.436222:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 00000020:00000040:2.0:1784948992.436224:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784948992.436227:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784948992.436229:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784948992.436229:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784948992.436230:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948992.436232:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948992.436238:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1784948992.436239:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000030a5bc8e. 00000100:00000001:3.0:1784948992.436239:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948992.436241:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948992.436248:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948992.436252:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948992.436265:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1784948992.436268:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948992.436269:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948992.436277:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784948992.436282:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948992.436283:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784948992.436286:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948992.436290:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948992.436293:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59e6b47] to pc [ptlrpcd_00_00+0] req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784948992.436298:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289666688:0@lo:400: 00000100:00000001:0.0:1784948992.436310:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784948992.436318:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784948992.436321:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948992.436323:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784948992.436328:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 00000100:00000001:2.0:1784948992.436329:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948992.436333:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948992.436339:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11269) 00000400:00000010:0.0:1784948992.436341:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001ee400ea. 00000400:00000010:0.0:1784948992.436351:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00000100:00000001:3.0:1784948992.436353:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948992.436357:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784948992.436358:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666688, portal 25 00000100:00000001:3.0:1784948992.436358:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948992.436363:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784948992.436364:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784948992.436365:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948992.436369:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000040:3.0:1784948992.436372:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784948992.436379:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948992.436402:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784948992.436403:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948992.436407:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948992.436412:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784948992.436416:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:0.0:1784948992.436417:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 02000000:00000001:3.0:1784948992.436420:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784948992.436422:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289666688, offset 0 02000000:00000001:3.0:1784948992.436424:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948992.436431:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289666816:192.168.202.137@tcp:400: 00000400:00000200:0.0:1784948992.436433:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784948992.436439:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948992.436442:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948992.436444:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948992.436446:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784948992.436449:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:3.0:1784948992.436450:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006cd4526f. 02000000:00000001:3.0:1784948992.436455:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948992.436461:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000400:00000200:0.0:1784948992.436465:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439480 00000400:00000010:3.0:1784948992.436468:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000400:00000200:0.0:1784948992.436474:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8069 [2] + 0 00000400:00000010:0.0:1784948992.436478:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002f32d921. 00000400:00000200:0.0:1784948992.436482:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000e4acfb45 00000100:00000200:3.0:1784948992.436487:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666816, portal 10 00000400:00000200:0.0:1784948992.436490:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784948992.436492:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784948992.436495:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529558303872 : -115544151247744 : ffff96e9c7e8b480) 00000400:00000200:0.0:1784948992.436497:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784948992.436505:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e4acfb45 00000100:00000040:3.0:1784948992.436508:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784948992.436509:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000e4acfb45. 00000100:00000001:0.0:1784948992.436517:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784948992.436520:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784948992.436522:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948992.436527:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948992.436529:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000100:00000200:3.0:1784948992.436532:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289666816, offset 0 00000100:00000040:0.0:1784948992.436535:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000046e93177 x1871651289666688 msgsize 224 00000400:00000200:3.0:1784948992.436541:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784948992.436548:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784948992.436550:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784948992.436556:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948992.436563:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784948992.436565:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784948992.436569:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784948992.436573:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784948992.436576:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784948992.436580:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3493:3493:241:4294967295] 192.168.202.137@tcp LPNI seq info [2249:2249:8:4294967295] 00000400:00000200:3.0:1784948992.436593:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784948992.436600:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000200:3.0:1784948992.436600:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784948992.436603:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000100:00100000:1.0:1784948992.436606:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666688 00000400:00000010:0.0:1784948992.436608:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 02000000:00000001:1.0:1784948992.436610:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000010:3.0:1784948992.436611:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000100:00000001:0.0:1784948992.436614:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784948992.436615:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784948992.436619:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948992.436624:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784948992.436625:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784948992.436625:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:1.0:1784948992.436634:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666688 00000020:00000001:1.0:1784948992.436640:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:3.0:1784948992.436643:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000040:1.0:1784948992.436645:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784948992.436646:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784948992.436652:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:1.0:1784948992.436653:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784948992.436654:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784948992.436658:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784948992.436666:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784948992.436672:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948992.436674:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.436676:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948992.436678:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784948992.436679:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948992.436686:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948992.436687:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000040:3.0:1784948992.436687:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2125 usecs (3 jiffies) 00000100:00000001:0.0:1784948992.436695:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305931818 00000100:00000001:1.0:1784948992.436696:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948992.436696:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305931779 00000020:00000010:1.0:1784948992.436705:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000010:0.0:1784948992.436707:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000020:00000010:1.0:1784948992.436713:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dd0652fb. 00000100:00000001:0.0:1784948992.436716:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:1.0:1784948992.436719:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008f9dc134. 00000100:00000001:0.0:1784948992.436725:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305931818 00000020:00000010:1.0:1784948992.436727:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784948992.436727:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305931818 00000100:00000001:3.0:1784948992.436735:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305931779 00000100:00000010:0.0:1784948992.436736:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000040:1.0:1784948992.436739:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784948992.436743:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784948992.436744:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948992.436749:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948992.436752:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948992.436753:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305931818 00000100:00000001:3.0:1784948992.436753:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305931818 00000100:00000001:1.0:1784948992.436761:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784948992.436763:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:3.0:1784948992.436763:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305931779 00000100:00000001:0.0:1784948992.436769:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784948992.436774:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948992.436776:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305931818 00000100:00000001:3.0:1784948992.436777:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305931818 00000100:00000001:3.0:1784948992.436785:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305931779 00000100:00000010:0.0:1784948992.436786:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:1.0:1784948992.436791:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784948992.436793:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784948992.436796:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784948992.436802:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948992.436802:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305931818 00000100:00100000:1.0:1784948992.436808:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 751 00000100:00000001:3.0:1784948992.436809:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948992.436811:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.436817:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948992.436819:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784948992.436820:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.436821:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948992.436827:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818138192 : -115538891413424 : ffff96eb016b6850) 00000100:00000001:3.0:1784948992.436828:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948992.436830:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.436838:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948992.436842:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784948992.436842:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000046e93177 x1871651289666688/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:653/0 lens 224/0 e 0 to 0 dl 1784949003 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948992.436843:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948992.436849:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784948992.436850:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948992.436853:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948992.436859:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948992.436859:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948992.436867:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948992.436867:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948992.436868:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784948992.436873:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784948992.436875:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:0.0:1784948992.436879:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784948992.436880:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784948992.436881:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1784948992.436886:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000020:00000010:1.0:1784948992.436889:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000800:00000001:3.0:1784948992.436890:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784948992.436893:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:1.0:1784948992.436901:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000046e93177 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289666688:12345-0@lo:400: 00000100:00000200:1.0:1784948992.436934:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666688 00000800:00000001:3.0:1784948992.436935:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948992.436938:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784948992.436941:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784948992.436942:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:2.0:1784948992.436945:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000020:00000001:1.0:1784948992.436948:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:2.0:1784948992.436950:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000800:00000001:3.0:1784948992.436950:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948992.436953:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948992.436958:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784948992.436959:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784948992.436964:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:3.0:1784948992.436968:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784948992.436969:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784948992.436971:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784948992.436976:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784948992.436981:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784948992.436985:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784948992.436990:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439500 00000100:00000001:2.0:1784948992.436991:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784948992.436992:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948992.436998:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:2.0:1784948992.437001:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948992.437003:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:3.0:1784948992.437007:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 163888 02000000:00000001:1.0:1784948992.437008:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784948992.437033:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000094500b89. 02000000:00000001:1.0:1784948992.437039:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948992.437040:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784948992.437048:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948992.437048:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784948992.437051:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948992.437055:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784948992.437056:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948992.437057:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000040:1.0:1784948992.437061:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666688 00010000:00000001:1.0:1784948992.437067:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:3.0:1784948992.437068:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000200:1.0:1784948992.437075:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000046e93177 x1871651289666688/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:653/0 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948992.437076:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948992.437081:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00010000:00000001:1.0:1784948992.437089:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000010:3.0:1784948992.437090:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000001218cfc. 00010000:00000001:1.0:1784948992.437092:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948992.437095:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000001218cfc x1871651289666816 msgsize 224 00000100:00001000:1.0:1784948992.437097:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=14 v=5 (1 1 1 1) 00000100:00000001:1.0:1784948992.437104:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1784948992.437106:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:1.0:1784948992.437110:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784948992.437114:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784948992.437116:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784948992.437118:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948992.437121:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948992.437123:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784948992.437125:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784948992.437130:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000800:00000001:3.0:1784948992.437133:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:1.0:1784948992.437135:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289666688, offset 224 00000800:00000001:3.0:1784948992.437140:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948992.437145:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948992.437154:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784948992.437156:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:2.0:1784948992.437158:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666816 02000000:00000001:2.0:1784948992.437162:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784948992.437165:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784948992.437168:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439480 00000100:00000001:2.0:1784948992.437169:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948992.437176:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784948992.437177:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8a91 [1] + 224 00000100:00100000:2.0:1784948992.437182:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666816 00000400:00000200:1.0:1784948992.437186:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784948992.437186:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784948992.437189:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784948992.437193:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784948992.437194:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000040:2.0:1784948992.437198:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:1.0:1784948992.437201:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1784948992.437202:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784948992.437207:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784948992.437210:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948992.437213:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784948992.437219:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000040:1.0:1784948992.437224:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948992.437224:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784948992.437231:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000020:00000010:2.0:1784948992.437241:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e5260462. 00000020:00000010:2.0:1784948992.437247:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000043a4433. 00000100:00000001:1.0:1784948992.437250:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784948992.437251:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000200:1.0:1784948992.437260:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000100:00000040:2.0:1784948992.437263:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:1.0:1784948992.437265:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000100:00000001:2.0:1784948992.437268:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784948992.437271:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948992.437272:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784948992.437274:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784948992.437276:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948992.437281:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.437282:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948992.437284:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784948992.437288:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784948992.437289:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784948992.437290:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784948992.437294:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784948992.437295:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:3.0:1784948992.437296:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1784948992.437299:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784948992.437302:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784948992.437303:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784948992.437307:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784948992.437308:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001ee400ea. 00000100:00000040:1.0:1784948992.437311:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046e93177 x1871651289666688/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:653/0 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948992.437316:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000100:00000001:2.0:1784948992.437320:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000010:3.0:1784948992.437322:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000100:00000001:2.0:1784948992.437324:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784948992.437327:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000046e93177 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289666688:12345-0@lo:400: Request processed in 455us (787us total) trans 0 rc 0/0 00000100:00000001:3.0:1784948992.437330:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784948992.437331:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:2.0:1784948992.437336:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 727 00000100:00000001:0.0:1784948992.437337:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:3.0:1784948992.437340:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948992.437343:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 751 00000100:00000040:2.0:1784948992.437345:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784948992.437346:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784948992.437353:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784948992.437353:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554950784 : -115544154600832 : ffff96e9c7b58a80) 00000100:00000001:0.0:1784948992.437354:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784948992.437362:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784948992.437366:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:2.0:1784948992.437369:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000001218cfc x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:653/0 lens 224/0 e 0 to 0 dl 1784949003 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948992.437369:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784948992.437373:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008f9dc134. 00000020:00000010:1.0:1784948992.437381:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784948992.437388:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dd0652fb. 00000020:00000040:1.0:1784948992.437395:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:1.0:1784948992.437401:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784948992.437405:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000094500b89. 00000100:00000001:3.0:1784948992.437410:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1784948992.437412:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784948992.437413:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948992.437414:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784948992.437417:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784948992.437419:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000001:3.0:1784948992.437419:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:1.0:1784948992.437423:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:3.0:1784948992.437424:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948992.437425:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948992.437429:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:1.0:1784948992.437430:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000014bfbbb1. 00000020:00000010:2.0:1784948992.437431:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000100:00000001:3.0:1784948992.437432:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948992.437439:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784948992.437440:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001ee8a23e. 00000100:00100000:2.0:1784948992.437445:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000001218cfc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289666816:12345-192.168.202.137@tcp:400: 00000100:00001000:3.0:1784948992.437449:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=14 v=5 (5 5 5 5) 00000100:00000200:2.0:1784948992.437450:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666816 00000020:00000001:2.0:1784948992.437455:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00001000:3.0:1784948992.437457:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=14 v=5 (1 1 1 1) 00000020:00000001:2.0:1784948992.437459:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784948992.437462:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:2.0:1784948992.437463:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.437465:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948992.437467:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784948992.437470:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00010000:00000001:3.0:1784948992.437470:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784948992.437474:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948992.437475:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784948992.437480:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.437480:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948992.437484:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784948992.437488:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:3.0:1784948992.437488:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784948992.437492:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948992.437497:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948992.437502:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784948992.437504:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948992.437505:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784948992.437509:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:2.0:1784948992.437511:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002b206cc5. 00000100:00000001:3.0:1784948992.437512:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948992.437517:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948992.437520:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948992.437522:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948992.437529:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784948992.437534:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666816 00000100:00100000:3.0:1784948992.437537:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b3c18d2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289666688:0@lo:400: 00010000:00000001:2.0:1784948992.437542:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784948992.437544:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.437548:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000200:2.0:1784948992.437554:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000001218cfc x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:653/0 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948992.437554:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b3c18d2b x1871651289666688/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948992.437576:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948992.437579:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948992.437583:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 00010000:00000001:2.0:1784948992.437585:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:3.0:1784948992.437588:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00010000:00000001:2.0:1784948992.437590:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948992.437592:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948992.437596:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00001000:2.0:1784948992.437598:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=9 v=5 (1 1 1 1) 00000020:00000001:3.0:1784948992.437599:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948992.437602:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000093457b1b. 00000100:00000001:2.0:1784948992.437607:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784948992.437610:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948992.437613:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1784948992.437615:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000010:3.0:1784948992.437616:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b3c18d2b. 00000100:00000001:3.0:1784948992.437622:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948992.437623:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784948992.437625:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948992.437631:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784948992.437632:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784948992.437636:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.437640:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948992.437642:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784948992.437645:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948992.437651:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784948992.437652:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784948992.437658:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:3.0:1784948992.437658:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784948992.437667:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289666816, offset 224 00000400:00000200:2.0:1784948992.437677:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784948992.437687:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784948992.437694:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3494:3494:241:4294967295] 192.168.202.137@tcp LPNI seq info [2250:2250:8:4294967295] 00000400:00000200:2.0:1784948992.437706:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784948992.437712:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784948992.437718:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000313a63a1. 00000800:00000200:2.0:1784948992.437723:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784948992.437727:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784948992.437731:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000313a63a1 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784948992.437745:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948992.437747:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784948992.437750:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784948992.437751:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784948992.437753:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784948992.437756:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000001218cfc x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:653/0 lens 224/224 e 0 to 0 dl 1784949003 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948992.437764:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000001218cfc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289666816:12345-192.168.202.137@tcp:400: Request processed in 352us (668us total) trans 0 rc 0/0 00000100:00100000:2.0:1784948992.437771:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 727 00000100:00000040:2.0:1784948992.437773:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784948992.437775:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784948992.437776:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784948992.437779:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000043a4433. 00000020:00000010:2.0:1784948992.437782:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784948992.437784:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e5260462. 00000020:00000040:2.0:1784948992.437788:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784948992.437790:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784948992.437933:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948992.437941:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784948992.437943:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784948992.437944:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784948992.437949:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000313a63a1. 00000800:00000001:2.0:1784948992.437950:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784948992.437958:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948992.437961:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784948992.437967:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948992.437972:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000200:2.0:1784948992.437977:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439500 00000400:00000010:0.0:1784948992.437978:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00000001:0.0:1784948992.437986:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948992.437991:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784948992.437991:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8a99 [1] + 224 00000800:00000001:2.0:1784948992.438001:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948992.438009:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948992.438012:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784948992.438033:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784948992.438037:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948992.438041:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784948992.438049:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948992.438062:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784948992.438067:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784948992.438070:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.438112:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948992.438116:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784948992.438121:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784948992.438122:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784948992.438123:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000400:00000200:3.0:1784948992.438130:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:3.0:1784948992.438131:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000001:3.0:1784948992.438134:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948992.438139:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784948992.438148:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948992.438155:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784948992.438156:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.438158:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784948992.438160:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784948992.438161:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948992.438164:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948992.438165:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948992.438169:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=9 v=5 (5 5 5 5) 00000100:00001000:3.0:1784948992.438172:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=9 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948992.438174:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948992.438175:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784948992.438176:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784948992.438179:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948992.438185:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.438188:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784948992.438190:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784948992.438192:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784948992.438194:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948992.438198:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948992.438207:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948992.438210:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784948992.438211:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784948992.438215:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948992.438224:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f59e6b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289666816:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784948992.438229:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.438232:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784948992.438235:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f59e6b47 x1871651289666816/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948992.438245:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784948992.438247:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784948992.438249:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006cd4526f. 02000000:00000001:3.0:1784948992.438253:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784948992.438255:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784948992.438257:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784948992.438261:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784948992.438263:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000030a5bc8e. 02000000:00000001:3.0:1784948992.438270:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784948992.438272:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784948992.438273:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f59e6b47. 00000100:00000001:3.0:1784948992.438277:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784948992.438277:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.438279:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948992.438283:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948992.438284:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948992.438286:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948992.438288:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948992.478988:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305931818, at 4305931819 00000100:00000010:0.0:1784948992.479005:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:3.0:1784948992.479068:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305931818, at 4305931819 00000100:00000040:1.0:1784948992.479070:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305931818, at 4305931819 00000100:00000010:3.0:1784948992.479086:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:1.0:1784948992.479090:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000040:2.0:1784948992.479105:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305931818, at 4305931819 00000100:00000010:2.0:1784948992.479120:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00010000:00000001:2.0:1784948993.268084:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948993.268094:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948993.268102:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948993.268103:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948993.268203:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948993.268211:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948993.268218:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948993.459997:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948993.460043:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948993.460047:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948993.460052:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948993.460056:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948993.460060:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948993.460063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948994.292036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948994.292044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948994.292051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948994.292052:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948994.292105:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948994.292111:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948994.292114:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948994.483988:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948994.484030:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948994.484032:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948994.484038:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948994.484041:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948994.484045:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948994.484048:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948995.316067:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948995.316077:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948995.316085:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948995.316087:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948995.316208:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948995.316215:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948995.316220:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948995.508020:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948995.508037:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948995.508040:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948995.508047:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948995.508051:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948995.508055:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948995.508058:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948996.340118:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948996.340128:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948996.340137:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948996.340140:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948996.340233:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948996.340241:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948996.340247:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948996.532025:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948996.532038:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948996.532040:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948996.532044:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948996.532048:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948996.532052:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948996.532055:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1784948997.044135:0:343958:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.26:988 00000400:00000100:2.0:1784948997.044146:0:343654:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.15:988 00000400:00000100:3.0:1784948997.044148:0:343954:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.13:988 00000400:00000100:1.0:1784948997.044229:0:343958:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.26@tcp at host 111.222.142.26:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784948997.044237:0:343954:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.13@tcp at host 111.222.142.13:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784948997.044238:0:343654:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.15@tcp at host 111.222.142.15:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784948997.044248:0:343958:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.26@tcp: down 00000400:00000200:3.0:1784948997.044284:0:343954:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.13@tcp: down 00000800:00000200:1.0:1784948997.044286:0:343958:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.142.26@tcp down 00000400:00000200:2.0:1784948997.044286:0:343654:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.15@tcp: down 00000400:00000200:3.0:1784948997.044296:0:343954:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1784948997.044307:0:343954:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000010:3.0:1784948997.044312:0:343954:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cee99234. 00000400:00000200:2.0:1784948997.044313:0:343654:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784948997.044325:0:343954:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:2.0:1784948997.044330:0:343654:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000800:00000010:2.0:1784948997.044337:0:343654:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000beaddf6f. 00000400:00000200:3.0:1784948997.044341:0:343954:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784948997.044350:0:343654:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784948997.044353:0:343954:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11270 health :900 00000400:00000200:2.0:1784948997.044363:0:343654:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784948997.044365:0:343954:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784948997.044371:0:343654:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784948997.044375:0:343954:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:2.0:1784948997.044384:0:343654:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:3.0:1784948997.044390:0:343954:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:2.0:1784948997.044418:0:343654:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:0.0:1784948997.044487:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784948997.044503:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:0.0:1784948997.044514:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948997.044518:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948997.044523:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:0.0:1784948997.044528:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:0.0:1784948997.044536:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948997.044543:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:0.0:1784948997.044550:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006200396a. 00000400:00000200:0.0:1784948997.044578:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:0.0:1784948997.044583:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:0.0:1784948997.044589:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:0.0:1784948997.044595:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:0.0:1784948997.044600:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784948997.044602:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784948997.044605:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:0.0:1784948997.044608:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:0.0:1784948997.044613:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784948997.044618:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:0.0:1784948997.044622:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000859865aa. 00000400:00000200:0.0:1784948997.044630:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:0.0:1784948997.044634:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:0.0:1784948997.044639:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784948997.364096:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948997.364107:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948997.364118:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948997.364122:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948997.364255:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948997.364262:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948997.364267:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784948997.556087:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948997.556097:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784948997.556110:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000546ff4bc. 00000020:00000040:0.0:1784948997.556118:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784948997.556126:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784948997.556127:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 02000000:00000001:0.0:1784948997.556131:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948997.556141:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.556142:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948997.556147:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784948997.556151:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000006487a147. 00000100:00000001:3.0:1784948997.556153:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948997.556160:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948997.556166:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.556167:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.556178:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948997.556188:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784948997.556211:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948997.556215:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948997.556226:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000546ff4bc] to pc [ptlrpcd_00_01+1] req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948997.556284:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784948997.556291:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784948997.556299:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784948997.556306:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784948997.556313:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784948997.556316:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784948997.556321:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000825eff83. 00000100:00000001:2.0:1784948997.556324:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948997.556325:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1784948997.556328:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784948997.556333:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784948997.556335:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948997.556336:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784948997.556343:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784948997.556349:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784948997.556357:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002aeee8e9. 00000100:00000001:0.0:1784948997.556363:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948997.556374:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1784948997.556380:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948997.556404:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784948997.556409:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.556415:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784948997.556419:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948997.556423:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000825eff83] to pc [ptlrpcd_00_02+2] req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784948997.556428:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784948997.556435:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784948997.556443:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948997.556449:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000546ff4bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289666944:0@lo:400: 00000100:00000001:3.0:1784948997.556456:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784948997.556464:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784948997.556467:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784948997.556471:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784948997.556472:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.556477:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784948997.556480:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000244a9cae. 00000100:00000001:2.0:1784948997.556483:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784948997.556487:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.556489:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:3.0:1784948997.556499:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000100:00000040:2.0:1784948997.556504:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784948997.556513:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000040:0.0:1784948997.556514:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11274) 00000100:00000001:2.0:1784948997.556531:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:3.0:1784948997.556531:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289666944, portal 25 00000100:00000001:2.0:1784948997.556536:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.556539:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1784948997.556544:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784948997.556545:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554975232 : -115544154576384 : ffff96e9c7b5ea00) 02000000:00000001:2.0:1784948997.556551:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948997.556557:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948997.556559:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948997.556569:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000825eff83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289667072:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948997.556587:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784948997.556595:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784948997.556596:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784948997.556602:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948997.556606:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784948997.556608:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:3.0:1784948997.556614:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 02000000:00000010:2.0:1784948997.556617:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 00000100:00000001:0.0:1784948997.556619:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1784948997.556624:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289666944, offset 0 02000000:00000001:2.0:1784948997.556627:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948997.556628:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948997.556642:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:2.0:1784948997.556643:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000603fb577. 00000400:00000010:2.0:1784948997.556653:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000100:00000200:2.0:1784948997.556660:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289667072, portal 10 00000100:00000001:2.0:1784948997.556670:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1784948997.556671:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784948997.556677:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554972544 : -115544154579072 : ffff96e9c7b5df80) 00000100:00000040:2.0:1784948997.556689:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784948997.556699:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439580 00000100:00000001:2.0:1784948997.556714:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784948997.556721:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x808d [2] + 0 00000100:00000040:2.0:1784948997.556727:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948997.556731:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000b3b2e3c. 00000400:00000010:2.0:1784948997.556737:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000400:00000200:3.0:1784948997.556741:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000a21d030b 00000400:00000200:3.0:1784948997.556754:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784948997.556756:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289667072, offset 0 00000400:00000200:3.0:1784948997.556768:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784948997.556773:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948997.556786:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a21d030b 00000400:00000200:2.0:1784948997.556794:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000010:3.0:1784948997.556797:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000a21d030b. 00000400:00000200:2.0:1784948997.556810:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784948997.556811:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784948997.556818:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1784948997.556822:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:3.0:1784948997.556828:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000470139eb x1871651289666944 msgsize 224 00000400:00000200:2.0:1784948997.556837:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00100000:3.0:1784948997.556851:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:2.0:1784948997.556854:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:3.0:1784948997.556859:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784948997.556870:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3495:3495:243:4294967295] 192.168.202.137@tcp LPNI seq info [2251:2251:8:4294967295] 00000100:00000001:3.0:1784948997.556891:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784948997.556946:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784948997.556951:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784948997.556959:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289666944 02000000:00000001:0.0:1784948997.556967:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000200:2.0:1784948997.556969:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784948997.556993:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784948997.557001:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784948997.557004:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000800:00000010:2.0:1784948997.557006:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005c3de731. 02000000:00000001:0.0:1784948997.557010:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784948997.557015:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 00000100:00100000:0.0:1784948997.557020:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289666944 00000800:00000200:2.0:1784948997.557028:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:0.0:1784948997.557030:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784948997.557032:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:0.0:1784948997.557038:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784948997.557044:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:3.0:1784948997.557049:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784948997.557051:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000040:0.0:1784948997.557052:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000800:00000200:2.0:1784948997.557062:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005c3de731 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:0.0:1784948997.557064:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784948997.557077:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784948997.557088:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784948997.557094:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.557096:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1784948997.557097:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000040:2.0:1784948997.557104:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2126 usecs (3 jiffies) 00000100:00000001:0.0:1784948997.557106:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784948997.557111:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948997.557116:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305936900 00000020:00000010:0.0:1784948997.557117:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000010:2.0:1784948997.557134:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a3d0cc2f. 00000020:00000010:0.0:1784948997.557138:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:3.0:1784948997.557143:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.557149:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000010:0.0:1784948997.557150:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007130446d. 00000100:00000001:3.0:1784948997.557152:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:0.0:1784948997.557164:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784948997.557166:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305936900 00000100:00000001:3.0:1784948997.557171:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948997.557179:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000445487c0. 00000100:00000001:3.0:1784948997.557181:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305936939 00000100:00000040:0.0:1784948997.557186:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784948997.557188:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784948997.557197:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305936900 00000100:00000001:0.0:1784948997.557199:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784948997.557203:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305936900 00000100:00000001:0.0:1784948997.557209:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000010:2.0:1784948997.557213:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001cae17e9. 00000100:00000001:3.0:1784948997.557214:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305936939 00000100:00000001:0.0:1784948997.557217:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.557224:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784948997.557228:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305936900 00000100:00000001:2.0:1784948997.557236:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305936900 00000100:00000001:0.0:1784948997.557237:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.557242:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305936939 00000100:00000010:2.0:1784948997.557246:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008a834f68. 00000100:00000001:2.0:1784948997.557256:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784948997.557258:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305936900 00000100:00000001:2.0:1784948997.557268:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948997.557270:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948997.557271:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305936939 00000100:00000001:2.0:1784948997.557278:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.557286:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.557289:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305936900 00000100:00000001:2.0:1784948997.557291:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.557305:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784948997.557308:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948997.557310:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784948997.557311:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784948997.557319:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784948997.557320:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.557321:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948997.557329:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784948997.557332:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.557335:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784948997.557337:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 752 00000100:00000001:3.0:1784948997.557344:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.557348:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948997.557350:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784948997.557356:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948997.557358:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818134096 : -115538891417520 : ffff96eb016b5850) 00000100:00000001:3.0:1784948997.557363:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:0.0:1784948997.557370:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000470139eb x1871651289666944/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:658/0 lens 224/0 e 0 to 0 dl 1784949008 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948997.557373:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784948997.557382:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948997.557389:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784948997.557393:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784948997.557396:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784948997.557404:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948997.557405:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784948997.557416:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00100000:0.0:1784948997.557427:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000470139eb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289666944:12345-0@lo:400: 00000100:00000200:0.0:1784948997.557433:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289666944 00000020:00000001:0.0:1784948997.557438:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784948997.557443:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784948997.557447:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948997.557450:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784948997.557453:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784948997.557458:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784948997.557461:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784948997.557464:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784948997.557467:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784948997.557469:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948997.557472:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784948997.557475:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784948997.557478:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784948997.557485:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006296bcc7. 02000000:00000001:0.0:1784948997.557488:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948997.557492:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784948997.557495:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784948997.557499:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289666944 00010000:00000001:0.0:1784948997.557503:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784948997.557508:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000470139eb x1871651289666944/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:658/0 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784948997.557519:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784948997.557523:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784948997.557528:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=19 v=5 (1 1 1 1) 00000800:00000200:2.0:1784948997.557530:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784948997.557536:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1784948997.557538:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005c3de731. 00000800:00000001:1.0:1784948997.557544:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784948997.557546:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:2.0:1784948997.557551:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784948997.557554:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784948997.557562:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000800:00000001:1.0:1784948997.557565:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948997.557567:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:0.0:1784948997.557570:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784948997.557578:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784948997.557582:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784948997.557586:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000800:00000001:1.0:1784948997.557588:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784948997.557594:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000100:00000040:0.0:1784948997.557597:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784948997.557608:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:0.0:1784948997.557609:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000400:00000200:1.0:1784948997.557614:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:0.0:1784948997.557621:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289666944, offset 224 00000100:00000200:2.0:1784948997.557623:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784948997.557641:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439600 00000400:00000200:0.0:1784948997.557642:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784948997.557660:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784948997.557669:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 164112 00000100:00000040:2.0:1784948997.557673:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784948997.557674:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:1.0:1784948997.557690:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784948997.557694:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.557701:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784948997.557702:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439580 00000800:00000001:1.0:1784948997.557704:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784948997.557718:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784948997.557724:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8aad [1] + 224 00000400:00000200:1.0:1784948997.557734:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784948997.557740:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784948997.557745:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:0.0:1784948997.557752:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:1.0:1784948997.557753:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784948997.557766:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000069457997. 00000100:00000001:0.0:1784948997.557767:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784948997.557776:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000069457997 x1871651289667072 msgsize 224 00000100:00000200:0.0:1784948997.557780:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784948997.557796:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:0.0:1784948997.557813:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784948997.557822:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:1.0:1784948997.557835:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784948997.557840:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948997.557842:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784948997.557858:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:0.0:1784948997.557864:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000100:00000001:0.0:1784948997.557873:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784948997.557876:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784948997.557885:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784948997.557893:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784948997.557898:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:0.0:1784948997.557899:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784948997.557905:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.557948:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948997.557956:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1784948997.557963:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289667072 00000020:00000001:0.0:1784948997.557965:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:2.0:1784948997.557967:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1784948997.557970:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784948997.557997:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784948997.558003:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784948997.558004:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1784948997.558005:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000470139eb x1871651289666944/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:658/0 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784948997.558021:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 02000000:00000001:3.0:1784948997.558022:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784948997.558035:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289667072 00000400:00000200:2.0:1784948997.558039:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000020:00000001:3.0:1784948997.558042:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784948997.558050:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:2.0:1784948997.558051:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000020:00000001:3.0:1784948997.558056:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:0.0:1784948997.558061:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000470139eb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289666944:12345-0@lo:400: Request processed in 661us (1222us total) trans 0 rc 0/0 00000020:00000040:3.0:1784948997.558065:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784948997.558066:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784948997.558076:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:2.0:1784948997.558083:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784948997.558085:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00100000:0.0:1784948997.558094:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 752 00000100:00000001:3.0:1784948997.558098:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:0.0:1784948997.558109:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000010:3.0:1784948997.558110:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000200:2.0:1784948997.558118:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784948997.558119:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948997.558121:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784948997.558128:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948997.558134:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000020:00000010:0.0:1784948997.558139:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007130446d. 00000020:00000010:3.0:1784948997.558149:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000070ada26c. 00000100:00000001:2.0:1784948997.558155:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784948997.558157:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:3.0:1784948997.558158:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b08eb9bf. 00000100:00000001:2.0:1784948997.558160:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784948997.558170:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000100:00000001:2.0:1784948997.558172:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:3.0:1784948997.558173:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 02000000:00000001:2.0:1784948997.558181:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:0.0:1784948997.558184:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784948997.558185:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784948997.558188:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784948997.558191:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.558197:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:2.0:1784948997.558198:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784948997.558199:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784948997.558203:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00001000:2.0:1784948997.558208:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=19 v=5 (5 5 5 5) 02000000:00000010:0.0:1784948997.558210:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006296bcc7. 00000100:00000001:3.0:1784948997.558210:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784948997.558227:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00001000:2.0:1784948997.558228:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=19 v=5 (1 1 1 1) 02000000:00000001:0.0:1784948997.558229:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784948997.558234:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784948997.558239:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.558240:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784948997.558241:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.558246:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948997.558255:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784948997.558258:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784948997.558259:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784948997.558264:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.558271:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784948997.558274:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cc579134. 00000100:00000040:2.0:1784948997.558275:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948997.558288:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000010:0.0:1784948997.558291:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f9492b64. 00000100:00000001:2.0:1784948997.558294:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784948997.558295:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784948997.558305:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948997.558308:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784948997.558315:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 728 00000100:00000040:2.0:1784948997.558322:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948997.558325:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784948997.558335:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549155328 : -115544160396288 : ffff96e9c75d1c00) 00000100:00100000:2.0:1784948997.558349:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000546ff4bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289666944:0@lo:400: 00000100:00000040:3.0:1784948997.558352:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000069457997 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:658/0 lens 224/0 e 0 to 0 dl 1784949008 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948997.558363:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.558371:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948997.558379:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000546ff4bc x1871651289666944/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948997.558381:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784948997.558389:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:2.0:1784948997.558393:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784948997.558396:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784948997.558398:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:3.0:1784948997.558404:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 02000000:00000010:2.0:1784948997.558405:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000244a9cae. 02000000:00000001:2.0:1784948997.558416:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1784948997.558423:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000069457997 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289667072:12345-192.168.202.137@tcp:400: 00000020:00000001:2.0:1784948997.558424:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948997.558429:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784948997.558437:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000200:3.0:1784948997.558439:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289667072 02000000:00000010:2.0:1784948997.558442:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000006487a147. 00000020:00000001:3.0:1784948997.558445:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:2.0:1784948997.558449:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1784948997.558452:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:2.0:1784948997.558454:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1784948997.558458:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784948997.558460:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000546ff4bc. 00000020:00000001:3.0:1784948997.558470:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784948997.558473:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784948997.558477:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:2.0:1784948997.558479:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1784948997.558481:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.558485:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948997.558487:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784948997.558491:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784948997.558493:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784948997.558499:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.558500:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1784948997.558502:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784948997.558506:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.558510:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784948997.558512:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784948997.558516:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784948997.558518:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784948997.558526:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003abbf7ac. 02000000:00000001:3.0:1784948997.558533:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.558535:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948997.558539:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784948997.558542:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289667072 00010000:00000001:3.0:1784948997.558545:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784948997.558550:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000069457997 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:658/0 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784948997.558561:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784948997.558564:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784948997.558567:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=14 v=5 (1 1 1 1) 00000100:00000001:3.0:1784948997.558571:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784948997.558574:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784948997.558577:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784948997.558580:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784948997.558582:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784948997.558585:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784948997.558588:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784948997.558591:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 00000100:00000200:3.0:1784948997.558595:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289667072, offset 224 00000400:00000200:3.0:1784948997.558601:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784948997.558611:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784948997.558617:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3496:3496:243:4294967295] 192.168.202.137@tcp LPNI seq info [2252:2252:8:4294967295] 00000400:00000200:3.0:1784948997.558629:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784948997.558635:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784948997.558639:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784948997.558645:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784948997.558651:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784948997.558655:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784948997.558672:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784948997.558676:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784948997.558680:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784948997.558682:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948997.558684:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784948997.558688:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000069457997 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:658/0 lens 224/224 e 0 to 0 dl 1784949008 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784948997.558703:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000069457997 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289667072:12345-192.168.202.137@tcp:400: Request processed in 318us (922us total) trans 0 rc 0/0 00000100:00100000:3.0:1784948997.558715:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 728 00000100:00000040:3.0:1784948997.558718:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784948997.558721:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784948997.558723:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784948997.558726:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b08eb9bf. 00000020:00000010:3.0:1784948997.558733:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784948997.558737:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000070ada26c. 00000020:00000040:3.0:1784948997.558742:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784948997.558744:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784948997.558823:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784948997.558826:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784948997.558834:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000800:00000001:3.0:1784948997.558839:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948997.558844:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784948997.558849:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948997.558859:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784948997.558860:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784948997.558874:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000400:00000010:2.0:1784948997.558883:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 00000400:00000200:3.0:1784948997.558884:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784948997.558894:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784948997.558900:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784948997.558962:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439600 00000400:00000200:3.0:1784948997.559000:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8ab5 [1] + 224 00000800:00000001:3.0:1784948997.559012:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948997.559023:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784948997.559027:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784948997.559033:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784948997.559040:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784948997.559047:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784948997.559060:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784948997.559082:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784948997.559090:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784948997.559094:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.559132:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948997.559137:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784948997.559142:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784948997.559144:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784948997.559147:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000603fb577. 00000400:00000200:2.0:1784948997.559153:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:2.0:1784948997.559155:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000100:00000001:2.0:1784948997.559158:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784948997.559165:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784948997.559175:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948997.559184:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784948997.559186:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.559188:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784948997.559190:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784948997.559192:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784948997.559194:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784948997.559197:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784948997.559202:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=14 v=5 (5 5 5 5) 00000100:00001000:2.0:1784948997.559206:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=14 v=5 (1 1 1 1) 00000100:00000001:2.0:1784948997.559209:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784948997.559211:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948997.559213:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784948997.559217:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784948997.559225:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.559229:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784948997.559230:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784948997.559232:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784948997.559234:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948997.559237:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948997.559245:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784948997.559248:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784948997.559249:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784948997.559253:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784948997.559261:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000825eff83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289667072:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784948997.559266:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.559268:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784948997.559270:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000825eff83 x1871651289667072/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784948997.559278:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784948997.559279:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784948997.559281:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 02000000:00000001:2.0:1784948997.559286:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784948997.559287:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784948997.559289:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784948997.559291:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784948997.559294:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002aeee8e9. 02000000:00000001:2.0:1784948997.559297:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784948997.559298:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784948997.559300:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000825eff83. 00000100:00000001:2.0:1784948997.559311:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784948997.559312:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.559314:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784948997.559319:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784948997.559320:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784948997.559323:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784948997.559325:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784948997.600025:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305936939, at 4305936940 00000100:00000010:2.0:1784948997.600038:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00000100:00000040:3.0:1784948997.600069:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305936939, at 4305936940 00000100:00000040:1.0:1784948997.600070:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305936939, at 4305936940 00000100:00000040:0.0:1784948997.600082:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305936939, at 4305936940 00000100:00000010:3.0:1784948997.600089:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008a834f68. 00000100:00000010:1.0:1784948997.600092:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000445487c0. 00000100:00000010:0.0:1784948997.600101:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a3d0cc2f. 00010000:00000001:2.0:1784948998.388069:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948998.388078:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948998.388089:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948998.388093:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784948998.388188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784948998.388194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784948998.388198:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948998.580141:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948998.580160:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948998.580163:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948998.580171:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948998.580174:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948998.580179:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948998.580182:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948999.412062:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784948999.412072:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784948999.412081:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784948999.412082:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784948999.412204:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784948999.412211:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784948999.412215:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784948999.604044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784948999.604061:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784948999.604063:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784948999.604069:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784948999.604073:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784948999.604077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784948999.604080:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949000.435993:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949000.435999:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949000.436004:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949000.436005:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949000.436046:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949000.436049:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949000.436052:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949000.628065:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949000.628081:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949000.628084:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949000.628090:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949000.628093:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949000.628098:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949000.628100:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949001.460056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949001.460065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949001.460073:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949001.460077:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949001.460080:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949001.460084:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949001.460087:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949001.460090:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949001.460092:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949001.460095:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949001.460192:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949001.460198:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949001.460201:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949001.652061:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949001.652075:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949001.652077:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949001.652083:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949001.652086:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949001.652091:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949001.652094:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949002.484058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949002.484068:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949002.484076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949002.484079:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949002.484189:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949002.484198:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949002.484205:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784949002.676172:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949002.676183:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949002.676193:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007bd58f9d. 00000020:00000040:1.0:1784949002.676202:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949002.676209:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949002.676213:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949002.676220:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949002.676231:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000001:1.0:1784949002.676241:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949002.676254:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949002.676269:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949002.676273:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949002.676282:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bd58f9d] to pc [ptlrpcd_00_03+3] req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949002.676321:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949002.676325:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949002.676330:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949002.676335:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949002.676339:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949002.676345:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ddd830de. 00000100:00000001:2.0:1784949002.676348:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784949002.676350:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00100000:2.0:1784949002.676358:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:1.0:1784949002.676360:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949002.676364:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949002.676367:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949002.676374:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949002.676377:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949002.676383:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949002.676385:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000061dbde97. 00000100:00000001:2.0:1784949002.676388:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949002.676395:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949002.676398:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949002.676399:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949002.676413:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949002.676418:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949002.676453:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949002.676459:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.676463:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949002.676465:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784949002.676467:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949002.676473:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.676475:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1784949002.676476:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949002.676486:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949002.676492:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289667200:0@lo:400: 00000100:00000040:1.0:1784949002.676495:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ddd830de] to pc [ptlrpcd_00_00+0] req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949002.676500:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784948977/real 1784948977] req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949002 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949002.676506:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949002.676516:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949002.676521:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949002.676531:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949002.676540:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 02000000:00000001:2.0:1784949002.676548:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949002.676561:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000603fb577. 00000400:00000001:0.0:1784949002.676575:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000010:2.0:1784949002.676575:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000001:0.0:1784949002.676582:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784949002.676587:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949002.676589:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000100:00000040:1.0:1784949002.676597:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11279) 00000400:00000200:0.0:1784949002.676601:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:0.0:1784949002.676606:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00000001:0.0:1784949002.676613:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949002.676617:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289667200, portal 25 00000100:00000001:1.0:1784949002.676618:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949002.676620:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949002 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949002.676626:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949002.676628:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949002.676635:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549161600 : -115544160390016 : ffff96e9c75d3480) 00000100:00000001:1.0:1784949002.676641:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949002.676649:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784949002.676652:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949002.676655:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949002.676671:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949002.676673:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949002 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949002.676675:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949002.676691:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949002.676698:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000100:00000200:2.0:1784949002.676708:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289667200, offset 0 00000400:00000200:2.0:1784949002.676730:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949002.676741:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949002.676745:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949002.676746:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.676754:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949002.676756:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784949002.676762:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:1.0:1784949002.676763:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949002.676765:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949002.676772:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949002.676776:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949002.676785:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949002 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949002.676787:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289667328:192.168.202.137@tcp:400: 00000400:00000200:2.0:1784949002.676795:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439680 00000100:00000001:1.0:1784949002.676808:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949002.676816:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1784949002.676818:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x80ad [2] + 0 02000000:00000001:1.0:1784949002.676824:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.676827:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949002.676829:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 02000000:00000001:1.0:1784949002.676838:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784949002.676841:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000042b6a6f7 00000100:00000001:0.0:1784949002.676843:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949002.676848:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949002.676851:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dd0652fb. 00000100:00000001:0.0:1784949002.676852:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949002.676853:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949002.676859:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784949002.676864:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.676866:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949002.676869:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000010:1.0:1784949002.676885:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002f6a97f7. 00000400:00000200:2.0:1784949002.676886:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000042b6a6f7 00000100:00000040:0.0:1784949002.676887:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949002 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949002.676896:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000042b6a6f7. 00000100:00000001:2.0:1784949002.676962:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949002.676966:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949002.676972:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000f3ed5d93 x1871651289667200 msgsize 224 00000400:00000010:1.0:1784949002.676974:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000100:00000001:0.0:1784949002.676987:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:1.0:1784949002.676989:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289667328, portal 10 00000100:00000001:0.0:1784949002.677001:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949002.677003:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:1.0:1784949002.677004:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949002.677010:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549156224 : -115544160395392 : ffff96e9c75d1f80) 00000100:00000001:0.0:1784949002.677012:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1784949002.677014:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949002.677020:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949002.677026:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:1.0:1784949002.677031:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949002.677032:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949002.677034:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00080000:0.0:1784949002.677047:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1784949002.677048:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000400:00000010:2.0:1784949002.677056:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000100:00000001:1.0:1784949002.677058:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949002.677070:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1784949002.677071:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949002.677079:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000200:2.0:1784949002.677080:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784949002.677084:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000200:1.0:1784949002.677088:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289667328, offset 0 00000100:00000001:0.0:1784949002.677098:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:1.0:1784949002.677111:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949002.677120:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949002 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949002.677123:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949002.677135:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949002.677136:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949002.677153:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1784949002.677154:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289665920:192.168.202.137@tcp:8: 00000400:00000200:1.0:1784949002.677167:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:2.0:1784949002.677180:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.677182:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949002.677183:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784949002.677187:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949002.677195:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784949002.677196:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:0.0:1784949002.677208:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289665920/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949002 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949002.677210:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3497:3497:243:4294967295] 192.168.202.137@tcp LPNI seq info [2253:2253:8:4294967295] 00000100:00000001:0.0:1784949002.677235:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949002.677241:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:1.0:1784949002.677243:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000010:0.0:1784949002.677246:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000008d04782c. 00000800:00000200:1.0:1784949002.677255:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784949002.677257:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1784949002.677259:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949002.677264:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000020:00000001:0.0:1784949002.677265:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:2.0:1784949002.677267:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305942059 00000020:00000040:0.0:1784949002.677272:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784949002.677280:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000010:2.0:1784949002.677281:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000010de4e21. 00000800:00000200:1.0:1784949002.677282:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000010:0.0:1784949002.677287:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000006e175381. 00000100:00000001:2.0:1784949002.677297:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1784949002.677299:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949002.677306:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:1.0:1784949002.677308:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000010:0.0:1784949002.677312:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 00000100:00000001:2.0:1784949002.677315:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305942059 00000800:00000200:1.0:1784949002.677316:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:2.0:1784949002.677328:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000100:00000001:0.0:1784949002.677329:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949002.677334:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949002.677339:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949002.677340:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949002.677342:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949002.677351:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305942059 00000100:00000001:0.0:1784949002.677355:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949002.677355:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2127 usecs (3 jiffies) 00000100:00000010:2.0:1784949002.677360:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e19b789f. 00000100:00000001:0.0:1784949002.677361:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677364:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305942020 00000100:00000001:2.0:1784949002.677373:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949002.677375:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949002.677376:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305942059 00000100:00000001:2.0:1784949002.677381:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305942059 00000100:00000001:0.0:1784949002.677383:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677385:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305942020 00000100:00000010:2.0:1784949002.677392:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e2039f57. 00000100:00000001:1.0:1784949002.677394:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305942059 00000100:00000001:2.0:1784949002.677401:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949002.677402:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305942020 00000100:00000001:1.0:1784949002.677411:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305942059 00000100:00000001:1.0:1784949002.677416:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305942020 00000100:00000001:2.0:1784949002.677416:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949002.677426:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677427:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305942059 00000100:00000001:2.0:1784949002.677434:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677435:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949002.677445:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677446:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677458:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677462:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949002.677469:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949002.677472:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949002.677473:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949002.677476:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949002.677479:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677481:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949002.677487:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.677492:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949002.677522:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949002.677535:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949002.677540:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949002.677544:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949002.677553:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949002.677561:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11249 00000400:00000200:0.0:1784949002.677572:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784949002.677579:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11248 00000400:00000200:0.0:1784949002.677606:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000800:00000200:1.0:1784949002.677610:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1784949002.677612:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000800:00000010:1.0:1784949002.677620:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000100:00000040:0.0:1784949002.677624:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000400:00000200:1.0:1784949002.677631:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949002.677632:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949002.677641:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000400:00000200:1.0:1784949002.677648:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949002.677650:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:1.0:1784949002.677658:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000020:00000001:0.0:1784949002.677659:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949002.677666:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000010:1.0:1784949002.677667:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000020:00000001:0.0:1784949002.677674:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949002.677678:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:0.0:1784949002.677681:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:0.0:1784949002.677687:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000200:1.0:1784949002.677688:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949002.677692:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:0.0:1784949002.677707:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949002.677715:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949002.677716:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949002.677722:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000040:1.0:1784949002.677728:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949002.677729:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:0.0:1784949002.677741:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000200:2.0:1784949002.677742:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:1.0:1784949002.677755:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.677760:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00080000:0.0:1784949002.677761:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000001:1.0:1784949002.677763:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1784949002.677774:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000800:00000001:1.0:1784949002.677781:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949002.677782:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949002.677784:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007a6587e5. 02000000:00000001:0.0:1784949002.677790:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1784949002.677794:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000800:00000001:1.0:1784949002.677797:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949002.677802:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949002.677805:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b9d99f33. 00000800:00000001:1.0:1784949002.677806:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:0.0:1784949002.677810:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.677817:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000001:1.0:1784949002.677821:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949002.677824:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.677824:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000010:0.0:1784949002.677839:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000400:00000200:1.0:1784949002.677843:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949002.677846:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000020:00000040:0.0:1784949002.677848:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784949002.677863:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1784949002.677864:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784949002.677869:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.677875:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949002.677877:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949002.677883:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1784949002.677885:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784949002.677889:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.677894:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000010:0.0:1784949002.677902:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006e175381. 00000400:00000200:2.0:1784949002.677975:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3498:3498:243:4294967295] 1.2.3.4@tcp LPNI seq info [106:106:8:4294967295] 00000400:00000200:2.0:1784949002.677987:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000001:0.0:1784949002.677995:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.678008:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00100000:0.0:1784949002.678014:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1784949002.678029:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000400:00000200:1.0:1784949002.678040:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439700 00000800:00000010:2.0:1784949002.678043:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005c3de731. 00000100:00000001:0.0:1784949002.678045:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949002.678051:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949002.678063:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_rcv+-1] req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949002.678064:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:1.0:1784949002.678071:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 164336 00000800:00000200:2.0:1784949002.678075:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 5 secs later) 00000800:00000001:1.0:1784949002.678100:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.1:1784949002.678101:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:2.1:1784949002.678109:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 5 secs later) 00000100:00000001:0.0:1784949002.678117:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949002.678118:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000800:00000001:1.0:1784949002.678123:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949002.678125:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005c3de731. 00000400:00000200:1.0:1784949002.678131:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784949002.678132:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11279) 00000400:00000200:2.0:1784949002.678137:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949002.678147:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949002.678156:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000001:1.0:1784949002.678158:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784949002.678159:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949002.678164:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000200:1.0:1784949002.678165:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949002.678168:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949002.678175:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949002.678176:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000010:1.0:1784949002.678178:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000053d87417. 00000100:00000040:0.0:1784949002.678188:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949002.678189:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000053d87417 x1871651289667328 msgsize 224 00000400:00000200:2.0:1784949002.678193:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:2.0:1784949002.678216:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:0.0:1784949002.678220:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1784949002.678222:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949002.678230:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.678230:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 02000000:00000001:0.0:1784949002.678241:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784949002.678247:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 02000000:00000001:0.0:1784949002.678248:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949002.678248:0:337113:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:0.0:1784949002.678258:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784949002.678259:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949002.678264:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000001:1.0:1784949002.678266:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:0.0:1784949002.678268:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289667456:1.2.3.4@tcp:8: 00000400:00000200:2.0:1784949002.678270:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000800:00000001:1.0:1784949002.678275:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949002.678278:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000100:00000001:0.0:1784949002.678281:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949002.678290:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.678292:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:0.0:1784949002.678294:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.678302:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949002.678302:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784949002.678307:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:0.0:1784949002.678313:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949002.678315:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949002.678315:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289667200 00000400:00000010:2.0:1784949002.678317:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007a6587e5. 00000100:00000001:0.0:1784949002.678324:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1784949002.678325:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289667328 02000000:00000001:3.0:1784949002.678327:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949002.678332:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949002.678334:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:2.0:1784949002.678335:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:3.0:1784949002.678336:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949002.678340:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949002.678342:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949002.678343:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.678347:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949002.678349:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.678350:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:0.0:1784949002.678355:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949002.678356:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949002.678361:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949002.678365:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:0.0:1784949002.678368:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949002.678371:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289667200 00000100:00100000:1.0:1784949002.678372:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289667328 00000020:00000001:1.0:1784949002.678382:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:3.0:1784949002.678383:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949002.678386:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000040:3.0:1784949002.678388:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949002.678392:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:3.0:1784949002.678394:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949002.678398:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000040:3.0:1784949002.678400:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949002.678406:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949002.678409:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784949002.678413:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949002.678418:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949002.678425:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949002.678427:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949002.678437:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000020:00000010:3.0:1784949002.678441:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:1.0:1784949002.678449:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949002.678450:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949002.678464:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000010:3.0:1784949002.678465:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000020:00000010:3.0:1784949002.678486:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000020:00000010:1.0:1784949002.678488:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000052e1e01a. 00000020:00000010:3.0:1784949002.678496:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000040c5ab13. 00000020:00000010:1.0:1784949002.678503:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f2b75cde. 00000020:00000010:3.0:1784949002.678507:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000020:00000010:1.0:1784949002.678519:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:3.0:1784949002.678525:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949002.678533:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949002.678537:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949002.678540:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949002.678545:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949002.678552:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.678554:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949002.678562:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949002.678569:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949002.678581:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949002.678586:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949002.678587:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949002.678594:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949002.678616:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949002.678622:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949002.678633:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 753 00000100:00000001:1.0:1784949002.678637:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949002.678638:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949002.678642:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949002.678645:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949002.678654:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818140240 : -115538891411376 : ffff96eb016b7050) 00000100:00000001:0.0:1784949002.678655:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.678664:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.678666:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949002.678675:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f3ed5d93 x1871651289667200/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:663/0 lens 224/0 e 0 to 0 dl 1784949013 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949002.678677:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949002.678696:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 729 00000100:00000040:1.0:1784949002.678707:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949002.678714:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949002.678716:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549159808 : -115544160391808 : ffff96e9c75d2d80) 00000020:00000010:3.0:1784949002.678723:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:3.0:1784949002.678730:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949002.678732:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000053d87417 x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:663/0 lens 224/0 e 0 to 0 dl 1784949013 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949002.678737:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000100:00100000:3.0:1784949002.678759:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f3ed5d93 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289667200:12345-0@lo:400: 00000100:00000001:1.0:1784949002.678770:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:3.0:1784949002.678770:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289667200 00000020:00000001:3.0:1784949002.678782:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000010:1.0:1784949002.678783:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000020:00000001:3.0:1784949002.678790:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949002.678793:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949002.678799:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949002.678803:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000001:3.0:1784949002.678808:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949002.678816:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00100000:1.0:1784949002.678827:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000053d87417 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289667328:12345-192.168.202.137@tcp:400: 00000020:00000001:3.0:1784949002.678828:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949002.678836:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949002.678842:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289667328 00000020:00000001:3.0:1784949002.678846:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949002.678852:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949002.678855:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949002.678862:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949002.678864:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949002.678872:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949002.678873:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949002.678879:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:1.0:1784949002.678882:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:3.0:1784949002.678885:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000001:1.0:1784949002.678887:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949002.678893:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 02000000:00000010:3.0:1784949002.678894:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000017348c4b. 00000020:00000001:1.0:1784949002.678897:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949002.678903:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949002.678954:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949002.678959:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949002.678964:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949002.678968:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949002.678975:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949002.678977:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949002.678981:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000001:3.0:1784949002.678985:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949002.678989:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a2676671. 00010000:00000040:3.0:1784949002.678992:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289667200 02000000:00000001:1.0:1784949002.678998:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949002.678999:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949002.679004:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949002.679010:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784949002.679011:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f3ed5d93 x1871651289667200/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:663/0 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1784949002.679016:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289667328 00010000:00000001:1.0:1784949002.679025:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949002.679036:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000053d87417 x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:663/0 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949002.679039:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949002.679044:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949002.679053:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=24 v=5 (1 1 1 1) 00010000:00000001:1.0:1784949002.679058:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949002.679062:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00010000:00000001:1.0:1784949002.679063:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949002.679069:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=19 v=5 (1 1 1 1) 00000100:00000040:3.0:1784949002.679070:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949002.679081:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784949002.679084:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784949002.679088:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:1.0:1784949002.679092:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:3.0:1784949002.679095:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.679098:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784949002.679101:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949002.679104:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949002.679109:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949002.679110:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784949002.679116:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:3.0:1784949002.679120:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000040:1.0:1784949002.679126:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:3.0:1784949002.679130:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289667200, offset 224 00000400:00000010:1.0:1784949002.679134:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000400:00000200:3.0:1784949002.679146:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000200:1.0:1784949002.679152:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289667328, offset 224 00000400:00000200:3.0:1784949002.679163:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784949002.679171:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949002.679180:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439680 00000400:00000200:1.0:1784949002.679190:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949002.679197:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8ac9 [1] + 224 00000400:00000200:1.0:1784949002.679205:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3499:3499:243:4294967295] 192.168.202.137@tcp LPNI seq info [2254:2254:8:4294967295] 00000400:00000200:3.0:1784949002.679214:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949002.679231:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949002.679235:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949002.679244:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949002.679250:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000100:00000001:3.0:1784949002.679253:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000200:1.0:1784949002.679259:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:3.0:1784949002.679267:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949002.679278:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949002.679288:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:3.0:1784949002.679304:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949002.679318:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949002.679325:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949002.679328:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949002.679331:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949002.679333:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1784949002.679334:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949002.679346:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000053d87417 x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:663/0 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949002.679348:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949002.679356:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:3.0:1784949002.679367:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949002.679373:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00100000:1.0:1784949002.679374:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000053d87417 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289667328:12345-192.168.202.137@tcp:400: Request processed in 600us (1178us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949002.679386:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949002.679395:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 729 00000100:00000040:3.0:1784949002.679396:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784949002.679400:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949002.679406:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00010000:00000001:3.0:1784949002.679408:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949002.679409:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949002.679416:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949002.679417:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000001:0.0:1784949002.679421:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784949002.679426:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000001:0.0:1784949002.679427:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1784949002.679427:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:0.0:1784949002.679436:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000603fb577. 00000020:00000010:1.0:1784949002.679438:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f2b75cde. 00000100:00000040:3.0:1784949002.679440:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f3ed5d93 x1871651289667200/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:663/0 lens 224/224 e 0 to 0 dl 1784949013 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949002.679447:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000020:00000010:1.0:1784949002.679451:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000400:00000010:0.0:1784949002.679460:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000020:00000010:1.0:1784949002.679463:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000052e1e01a. 00000100:00000001:0.0:1784949002.679479:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1784949002.679479:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f3ed5d93 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289667200:12345-0@lo:400: Request processed in 764us (2502us total) trans 0 rc 0/0 00000020:00000040:1.0:1784949002.679481:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949002.679491:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949002.679498:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949002.679504:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 753 00000800:00000200:2.0:1784949002.679506:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:3.0:1784949002.679515:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000800:00000010:2.0:1784949002.679518:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000100:00000001:3.0:1784949002.679523:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949002.679534:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:1.0:1784949002.679538:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949002.679539:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949002.679542:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000040c5ab13. 00000100:00000200:0.0:1784949002.679552:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949002.679553:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949002.679556:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000800:00000001:1.0:1784949002.679566:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949002.679566:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949002.679570:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000800:00000001:1.0:1784949002.679576:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784949002.679583:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:2.0:1784949002.679588:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 02000000:00000001:3.0:1784949002.679596:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000400:00000010:2.0:1784949002.679599:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000400:00000200:1.0:1784949002.679601:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000010:3.0:1784949002.679603:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000017348c4b. 00000100:00000001:2.0:1784949002.679616:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949002.679617:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1784949002.679618:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949002.679626:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949002.679627:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439700 00000100:00000001:0.0:1784949002.679628:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949002.679628:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949002.679640:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:3.0:1784949002.679644:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:0.0:1784949002.679647:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000200:1.0:1784949002.679652:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8ad1 [1] + 224 00000100:00000001:0.0:1784949002.679654:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:3.0:1784949002.679658:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000100:00000001:0.0:1784949002.679661:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949002.679665:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784949002.679669:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c9198cec. 02000000:00000001:0.0:1784949002.679670:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949002.679679:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784949002.679683:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=24 v=5 (5 5 5 5) 00000400:00000200:1.0:1784949002.679685:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00001000:0.0:1784949002.679697:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=24 v=5 (1 1 1 1) 00000400:00000200:1.0:1784949002.679700:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949002.679710:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949002.679713:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949002.679715:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949002.679722:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:1.0:1784949002.679723:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949002.679726:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949002.679736:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949002.679750:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949002.679757:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949002.679790:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949002.679798:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949002.679800:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949002.679806:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949002.679811:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:0.0:1784949002.679820:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007bd58f9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289667200:0@lo:400: 00000800:00000001:1.0:1784949002.679823:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949002.679831:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949002.679833:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949002.679840:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949002.679847:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007bd58f9d x1871651289667200/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949002.679858:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949002.679863:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1784949002.679865:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949002.679866:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 00000100:00000001:2.0:1784949002.679873:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949002.679877:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949002.679882:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000001:2.0:1784949002.679884:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:0.0:1784949002.679885:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000001:2.0:1784949002.679893:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1784949002.679894:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949002.679898:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 00000400:00000010:2.0:1784949002.679902:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002f6a97f7. 02000000:00000001:0.0:1784949002.679973:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949002.679977:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949002.679979:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007bd58f9d. 00000100:00000001:0.0:1784949002.679986:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949002.679992:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949002.679996:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949002.679997:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000100:00000001:0.0:1784949002.680003:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:2.0:1784949002.680004:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000100:00000001:0.0:1784949002.680009:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949002.680015:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949002.680016:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949002.680025:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949002.680031:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949002.680053:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949002.680069:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949002.680073:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949002.680077:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949002.680081:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949002.680084:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949002.680087:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949002.680091:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949002.680100:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=19 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949002.680107:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=19 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949002.680112:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949002.680114:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949002.680118:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949002.680125:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949002.680141:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949002.680146:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949002.680149:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949002.680153:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949002.680156:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949002.680162:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949002.680178:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949002.680183:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949002.680185:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949002.680191:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949002.680206:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ddd830de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289667328:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949002.680214:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949002.680218:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949002.680223:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ddd830de x1871651289667328/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949018 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949002.680249:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949002.680252:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949002.680256:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dd0652fb. 02000000:00000001:2.0:1784949002.680262:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949002.680265:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949002.680267:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949002.680271:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949002.680275:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000061dbde97. 02000000:00000001:2.0:1784949002.680280:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949002.680283:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949002.680286:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ddd830de. 00000100:00000001:2.0:1784949002.680292:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949002.680295:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949002.680298:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949002.680306:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949002.680308:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949002.680312:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949002.680315:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949002.720018:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305942059, at 4305942060 00000100:00000010:1.0:1784949002.720032:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00000100:00000040:3.0:1784949002.720057:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305942059, at 4305942060 00000100:00000040:0.0:1784949002.720073:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305942059, at 4305942060 00000100:00000010:3.0:1784949002.720074:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00000100:00000040:2.0:1784949002.720075:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305942059, at 4305942060 00000100:00000010:0.0:1784949002.720093:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000010de4e21. 00000100:00000010:2.0:1784949002.720097:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e19b789f. 00010000:00000001:2.0:1784949003.508084:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949003.508093:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949003.508101:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949003.508103:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949003.508221:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949003.508229:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949003.508234:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949003.700097:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949003.700108:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949003.700125:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949002/real 0] req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949003.700145:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.700149:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949003.700151:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949003.700157:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949003.700168:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.700179:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949003.700182:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949003.700186:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949003.700190:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949003.700192:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949003.700198:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949003 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949003.700214:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949003.700220:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.700224:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949003.700228:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.700233:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949003.700237:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949003.700245:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949003.700284:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:2.0:1784949003.700292:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949003.700301:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949003 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949003.700313:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289667456:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949003.700324:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.700329:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949003.700335:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289667456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949003 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949003.700340:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949003.700348:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784949003.700352:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000020:00000001:2.0:1784949003.700356:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949003.700361:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000010:0.0:1784949003.700363:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000617daa1a. 00000020:00000001:2.0:1784949003.700365:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000040:0.0:1784949003.700375:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 02000000:00000010:2.0:1784949003.700375:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000006e175381. 00000100:00000001:0.0:1784949003.700386:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949003.700390:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949003.700393:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949003.700395:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949003.700400:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 02000000:00000001:0.0:1784949003.700401:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949003.700414:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000eafb2cdc. 00000100:00000001:2.0:1784949003.700415:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949003.700421:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.700424:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949003.700425:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949003.700430:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949003.700435:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949003.700438:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949003.700441:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949003.700442:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.700468:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949003.700474:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949003.700484:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000617daa1a] to pc [ptlrpcd_00_01+1] req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949003.700545:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949003.700551:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949003.700559:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949003.700564:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949003.700568:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784949003.700571:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949003.700574:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1784949003.700577:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949003.700583:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1784949003.700587:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11249 00000100:00000040:2.0:1784949003.700595:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949003.700597:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949003.700607:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:3.0:1784949003.700609:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949003.700623:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11274 00000100:00000001:2.0:1784949003.700629:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949003.700634:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949003.700639:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949003.700642:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949003.700647:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949003.700650:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00100000:2.0:1784949003.700655:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289667584:0@lo:400: 00000400:00000200:0.0:1784949003.700662:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:2.0:1784949003.700667:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1784949003.700669:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1784949003.700677:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784949003.700678:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000400:00000200:3.0:1784949003.700683:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 02000000:00000001:2.0:1784949003.700684:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.700686:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949003.700695:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784949003.700700:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000400:00000200:3.0:1784949003.700700:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 02000000:00000010:2.0:1784949003.700705:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007a6587e5. 00000100:00000001:0.0:1784949003.700712:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949003.700713:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949003.700718:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000020:00000001:0.0:1784949003.700725:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949003.700730:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000010:2.0:1784949003.700731:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001d3d7473. 00000400:00000010:3.0:1784949003.700732:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000020:00000001:0.0:1784949003.700735:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949003.700744:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:2.0:1784949003.700751:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000020:00000001:0.0:1784949003.700752:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:3.0:1784949003.700756:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000092c40cf9. 00000020:00000001:0.0:1784949003.700763:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000200:2.0:1784949003.700767:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289667584, portal 25 00000100:00000040:0.0:1784949003.700775:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949003.700777:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1784949003.700781:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:2.0:1784949003.700784:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554966272 : -115544154585344 : ffff96e9c7b5c700) 00000100:00000001:0.0:1784949003.700786:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949003.700795:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000040:2.0:1784949003.700799:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949003.700801:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000200:3.0:1784949003.700812:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000020:00000040:0.0:1784949003.700813:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:0.0:1784949003.700829:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000001:2.0:1784949003.700830:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949003.700836:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1784949003.700844:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000040:2.0:1784949003.700846:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:3.0:1784949003.700855:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949003.700857:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949003.700857:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 02000000:00000001:0.0:1784949003.700868:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:2.0:1784949003.700869:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289667584, offset 0 00000400:00000200:3.0:1784949003.700870:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784949003.700881:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949003.700886:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000400:00000200:2.0:1784949003.700891:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:0.0:1784949003.700894:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.700901:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784949003.700975:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949003.700983:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3500:3500:243:4294967295] 1.2.3.4@tcp LPNI seq info [107:107:8:4294967295] 00000100:00000010:0.0:1784949003.700984:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949003.700992:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784949003.701003:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949003.701006:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949003.701011:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000001:0.0:1784949003.701013:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949003.701021:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949003.701025:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949003.701035:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000b91d4ef5. 00000400:00000200:3.0:1784949003.701037:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:0.0:1784949003.701046:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949003.701061:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00100000:0.0:1784949003.701062:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949003.701071:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:3.0:1784949003.701079:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000100:00000001:0.0:1784949003.701093:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1784949003.701096:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439800 00000100:00000001:0.0:1784949003.701098:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949003.701108:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:0.0:1784949003.701117:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_rcv+-1] req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949003.701120:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x80cd [2] + 0 00000800:00000200:3.0:1784949003.701121:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 4 secs later) 00000400:00000010:2.0:1784949003.701138:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c6713e8b. 00000400:00000200:2.0:1784949003.701152:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000092ad473a 00000800:00000200:3.1:1784949003.701154:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.1:1784949003.701165:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 4 secs later) 00000400:00000200:2.0:1784949003.701166:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949003.701170:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949003.701182:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1784949003.701191:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000100:3.0:1784949003.701193:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784949003.701198:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000800:00000010:3.0:1784949003.701203:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000100:00000010:0.0:1784949003.701208:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d003ffea. 00000400:00000200:2.0:1784949003.701211:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000092ad473a 00000400:00000200:3.0:1784949003.701213:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:0.0:1784949003.701217:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000010:2.0:1784949003.701223:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000092ad473a. 00000100:00000001:0.0:1784949003.701225:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949003.701231:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1784949003.701234:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 02000000:00000001:0.0:1784949003.701237:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949003.701240:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:3.0:1784949003.701243:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 02000000:00000010:0.0:1784949003.701247:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b51ad697. 00000100:00000200:2.0:1784949003.701249:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1784949003.701256:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000001:0.0:1784949003.701258:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949003.701262:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f24717d4 x1871651289667584 msgsize 224 00000100:00000040:0.0:1784949003.701275:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1784949003.701282:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:1.0:1784949003.701286:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949003.701293:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:1.0:1784949003.701297:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949003.701303:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949003.701307:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949003.701315:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:1.0:1784949003.701318:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949003.701323:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949003.701328:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949003.701334:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000001:2.0:1784949003.701335:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784949003.701345:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d003ffea] to pc [ptlrpcd_00_02+2] req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949003.701356:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000200:3.0:1784949003.701361:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000010:2.0:1784949003.701367:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949003.701380:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:3.0:1784949003.701387:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:2.0:1784949003.701389:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784949003.701393:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949003.701397:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949003.701401:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949003.701412:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000040:0.0:1784949003.701413:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11280) 00000100:00000001:1.0:1784949003.701422:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949003.701429:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949003.701430:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:2.0:1784949003.701439:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784949003.701442:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784949003.701447:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000001:1.0:1784949003.701449:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949003.701450:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949003.701458:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:3.0:1784949003.701459:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 02000000:00000001:1.0:1784949003.701460:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949003.701470:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289667584 00000100:00100000:1.0:1784949003.701479:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289667712:192.168.202.137@tcp:8: 00000400:00000200:3.0:1784949003.701480:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 02000000:00000001:0.0:1784949003.701482:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949003.701491:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.701492:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949003.701494:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784949003.701495:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:0.0:1784949003.701503:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949003.701503:0:337120:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1784949003.701505:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:3.0:1784949003.701509:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 02000000:00000001:1.0:1784949003.701512:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949003.701519:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949003.701520:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949003.701528:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949003.701531:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289667584 00000100:00000001:2.0:1784949003.701538:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305943083 00000020:00000001:0.0:1784949003.701540:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949003.701548:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:2.0:1784949003.701551:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e19b789f. 00000020:00000001:0.0:1784949003.701553:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949003.701560:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949003.701564:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1784949003.701566:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784949003.701574:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949003.701578:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949003.701580:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305943083 02000000:00000010:1.0:1784949003.701581:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d58187af. 00000100:00000001:0.0:1784949003.701582:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:3.0:1784949003.701587:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 02000000:00000001:1.0:1784949003.701593:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949003.701594:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784949003.701594:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784949003.701595:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001cae17e9. 00000100:00000001:0.0:1784949003.701608:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784949003.701609:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000085e44521. 00000100:00000001:2.0:1784949003.701611:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949003.701614:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784949003.701623:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784949003.701624:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.701625:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305943083 00000400:00000010:1.0:1784949003.701628:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000100:00000001:3.0:1784949003.701639:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:2.0:1784949003.701640:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000ef8f137f. 00000100:00000200:1.0:1784949003.701641:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289667712, portal 4 00000020:00000010:0.0:1784949003.701644:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:3.0:1784949003.701645:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949003.701652:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949003.701654:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949003.701655:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000010:0.0:1784949003.701658:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007130446d. 00000100:00000001:2.0:1784949003.701659:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305943083 00000100:00000001:1.0:1784949003.701662:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000100:00000040:3.0:1784949003.701670:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949003.701672:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000010:2.0:1784949003.701675:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a2179890. 00000100:00000040:1.0:1784949003.701683:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949003.701688:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000040:0.0:1784949003.701692:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949003.701700:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949003.701704:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949003.701711:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949003.701714:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.701719:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949003.701728:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.701733:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949003.701737:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949003.701739:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.701741:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949003.701746:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949003.701754:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784949003.701757:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949003.701758:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784949003.701760:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949003.701763:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.701766:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000010:1.0:1784949003.701769:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 02000000:00000001:3.0:1784949003.701775:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949003.701780:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289667712, offset 0 00000100:00000001:2.0:1784949003.701780:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949003.701786:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1784949003.701790:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000d003ffea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289667840:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949003.701792:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949003.701793:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.701799:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949003.701804:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949003.701805:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949003.701812:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1784949003.701816:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 754 02000000:00000001:3.0:1784949003.701823:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784949003.701824:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:1.0:1784949003.701832:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000001:3.0:1784949003.701833:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.701834:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457130576 : -115544252421040 : ffff96e9c1e0ec50) 00000100:00000001:2.0:1784949003.701844:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949003.701848:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949003.701852:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949003.701854:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f24717d4 x1871651289667584/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:664/0 lens 224/0 e 0 to 0 dl 1784949014 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949003.701856:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:3.0:1784949003.701862:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 00000400:00000200:1.0:1784949003.701871:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:3.0:1784949003.701872:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949003.701886:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000df521cee. 00000400:00000200:1.0:1784949003.701888:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949003.701889:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949003.701898:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000400:00000010:3.0:1784949003.701900:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000001:0.0:1784949003.701965:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949003.701970:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000200:3.0:1784949003.701973:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289667840, portal 10 00000400:00000200:1.0:1784949003.701984:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949003.701993:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949003.701997:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554964480 : -115544154587136 : ffff96e9c7b5c000) 00000100:00100000:0.0:1784949003.702006:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f24717d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289667584:12345-0@lo:400: 00000400:00000200:1.0:1784949003.702009:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3501:3501:243:4294967295] 192.168.202.137@tcp LPNI seq info [2255:2255:8:4294967295] 00000100:00000200:0.0:1784949003.702017:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289667584 00000100:00000040:3.0:1784949003.702021:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949003.702028:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949003.702037:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949003.702041:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1784949003.702045:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949003.702053:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:3.0:1784949003.702058:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949003.702060:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:1.0:1784949003.702067:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000400:00000010:3.0:1784949003.702068:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000467f058f. 00000020:00000001:0.0:1784949003.702071:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949003.702078:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949003.702080:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289667840, offset 0 00000020:00000001:0.0:1784949003.702088:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000200:1.0:1784949003.702095:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949003.702096:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784949003.702098:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949003.702103:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949003.702110:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000010:1.0:1784949003.702113:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000adf08f9b. 00000100:00000001:0.0:1784949003.702114:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:3.0:1784949003.702120:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000001:0.0:1784949003.702122:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:3.0:1784949003.702131:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:0.0:1784949003.702134:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006e175381. 00000800:00000200:1.0:1784949003.702137:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784949003.702144:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:0.0:1784949003.702146:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949003.702156:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949003.702158:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949003.702167:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:3.0:1784949003.702167:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:0.0:1784949003.702171:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949003.702181:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 1, nob 616 niov 1 nkiov 1 00000400:00000200:3.0:1784949003.702182:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3502:3502:242:4294967295] 192.168.202.137@tcp LPNI seq info [2256:2256:7:4294967295] 00010000:00000040:0.0:1784949003.702184:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289667584 00010000:00000001:0.0:1784949003.702199:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949003.702215:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f24717d4 x1871651289667584/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:664/0 lens 224/224 e 0 to 0 dl 1784949014 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949003.702216:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949003.702223:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949003.702238:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949003.702239:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000800:00000010:3.0:1784949003.702250:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000001:1.0:1784949003.702253:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305943084 00000800:00000200:3.0:1784949003.702269:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00010000:00000001:0.0:1784949003.702272:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949003.702273:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305943083 00010000:00000001:0.0:1784949003.702282:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949003.702285:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949003.702288:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305943084 00000100:00001000:0.0:1784949003.702291:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=25 v=5 (1 1 1 1) 00000800:00000200:3.0:1784949003.702295:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949003.702300:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305943083 00000100:00000001:0.0:1784949003.702310:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949003.702318:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305943084 00000100:00000040:0.0:1784949003.702319:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949003.702328:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949003.702329:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784949003.702331:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305943083 02000000:00000001:0.0:1784949003.702343:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:3.0:1784949003.702343:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2131 usecs (3 jiffies) 00000100:00000001:1.0:1784949003.702346:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305943084 02000000:00000001:0.0:1784949003.702351:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.702356:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305943045 00000100:00000001:1.0:1784949003.702358:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305943083 00000100:00000001:0.0:1784949003.702360:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949003.702364:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305943084 00000100:00000001:1.0:1784949003.702371:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949003.702373:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949003.702376:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305943045 00000100:00000001:1.0:1784949003.702382:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.702384:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305943084 00000400:00000010:0.0:1784949003.702386:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00000001:1.0:1784949003.702388:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.702391:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305943045 00000100:00000001:1.0:1784949003.702399:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.702400:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305943084 00000100:00000200:0.0:1784949003.702401:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289667584, offset 224 00000100:00000001:3.0:1784949003.702413:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305943045 00000100:00000001:1.0:1784949003.702415:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:2.0:1784949003.702417:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949003.702420:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305943084 00000100:00000001:1.0:1784949003.702422:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1784949003.702428:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000010:2.0:1784949003.702429:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000adf08f9b. 00000100:00000001:3.0:1784949003.702433:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949003.702435:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949003.702442:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949003.702443:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949003.702444:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949003.702446:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.702452:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949003.702457:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.702461:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949003.702464:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949003.702468:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949003.702484:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:2.0:1784949003.702487:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000800:00000001:3.0:1784949003.702490:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949003.702494:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784949003.702498:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439800 00000100:00000200:2.0:1784949003.702505:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949003.702510:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949003.702521:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949003.702524:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8aed [1] + 224 00000100:00000001:2.0:1784949003.702542:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949003.702548:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949028 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949003.702552:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949003.702552:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949003.702572:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949003.702578:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949003.702580:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949003.702581:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949003.702613:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:3.0:1784949003.702621:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6439880 00000100:00000200:0.0:1784949003.702628:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000100:3.0:1784949003.702640:0:337114:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289667712 offset 0 length 520: 4 00000400:00000200:3.0:1784949003.702652:0:337114:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000003d00254a is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:3.0:1784949003.702661:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000040:0.0:1784949003.702666:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949003.702677:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949003.702685:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949003.702693:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949003.702700:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949003.702707:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949003.702710:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000800:00000200:1.0:1784949003.702714:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949003.702715:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949003.702717:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000800:00000010:1.0:1784949003.702729:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000800:00000001:3.0:1784949003.702730:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949003.702731:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949003.702740:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949003.702743:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949003.702744:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949003.702746:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949003.702752:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949003.702753:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949003.702770:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:1.0:1784949003.702772:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949003.702772:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439900 00010000:00000001:0.0:1784949003.702782:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949003.702787:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949003.702796:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 164560 00000020:00000001:0.0:1784949003.702797:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784949003.702799:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949003.702808:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f24717d4 x1871651289667584/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:664/0 lens 224/224 e 0 to 0 dl 1784949014 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949003.702809:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949003.702811:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000800:00000001:2.0:1784949003.702812:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784949003.702823:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000100:00100000:0.0:1784949003.702833:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f24717d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289667584:12345-0@lo:400: Request processed in 941us (1567us total) trans 0 rc 0/0 00000800:00000001:2.0:1784949003.702834:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949003.702838:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949003.702847:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000001:3.0:1784949003.702847:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:1.0:1784949003.702856:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949003.702859:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 754 00000400:00000001:3.0:1784949003.702859:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949003.702869:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001d3d7473. 00000400:00000200:2.0:1784949003.702870:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:0.0:1784949003.702872:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949003.702884:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000200:3.0:1784949003.702887:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000100:00000001:0.0:1784949003.702890:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000010:3.0:1784949003.702896:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000100:00000001:1.0:1784949003.702897:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1784949003.702901:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007130446d. 00000100:00000001:3.0:1784949003.702971:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784949003.702973:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949003.702974:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949003.702979:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949003.702985:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949003.702987:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000100:00000200:2.0:1784949003.702988:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000040:0.0:1784949003.703007:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000010:2.0:1784949003.703007:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59e6b47. 00000100:00000040:2.0:1784949003.703021:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000f59e6b47 x1871651289667840 msgsize 224 02000000:00000001:0.0:1784949003.703024:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949003.703025:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949003.703033:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006e175381. 00000100:00000001:1.0:1784949003.703036:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784949003.703046:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:0.0:1784949003.703047:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000200:3.0:1784949003.703051:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949003.703055:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949003.703069:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784949003.703070:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949003.703073:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949003.703081:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949003.703083:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000152ac2d7. 00000100:00000001:3.0:1784949003.703086:0:337120:0:(client.c:1580:after_reply()) Process entered 00000800:00000001:2.0:1784949003.703088:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949003.703093:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:0.0:1784949003.703095:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004ba703f8. 00000100:00000001:3.0:1784949003.703098:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000001:2.0:1784949003.703099:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949003.703101:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949003.703105:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949003.703110:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289667840 02000000:00000001:3.0:1784949003.703112:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949003.703116:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949003.703121:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00001000:3.0:1784949003.703121:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=25 v=5 (5 5 5 5) 00000100:00000001:1.0:1784949003.703124:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949003.703137:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949003.703138:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=25 v=5 (1 1 1 1) 00000100:00100000:1.0:1784949003.703148:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289667840 00000100:00000001:3.0:1784949003.703150:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:1.0:1784949003.703154:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949003.703156:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949003.703159:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000001:3.0:1784949003.703163:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:1.0:1784949003.703164:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00010000:00000001:3.0:1784949003.703169:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949003.703171:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:3.0:1784949003.703175:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949003.703177:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949003.703184:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:3.0:1784949003.703185:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949003.703190:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949003.703198:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000a400f603. 00000100:00000001:3.0:1784949003.703202:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949003.703205:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949003.703208:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:1.0:1784949003.703212:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000a400f603. 00000100:00000001:3.0:1784949003.703214:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949003.703222:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949003.703223:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000052e1e01a. 00000020:00000010:1.0:1784949003.703235:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f2b75cde. 00000020:00000010:1.0:1784949003.703244:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00100000:3.0:1784949003.703247:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289667584:0@lo:400: 00000100:00000040:1.0:1784949003.703260:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949003.703261:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949003.703271:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949003.703272:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949003.703279:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:3.0:1784949003.703281:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000617daa1a x1871651289667584/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949003.703285:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949003.703299:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949003.703301:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1784949003.703305:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:3.0:1784949003.703307:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1784949003.703309:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949003.703312:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007a6587e5. 02000000:00000001:3.0:1784949003.703324:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949003.703326:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949003.703328:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949003.703330:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949003.703333:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000eafb2cdc. 00000100:00000001:1.0:1784949003.703334:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949003.703341:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949003.703346:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949003.703348:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000617daa1a. 00000100:00000001:1.0:1784949003.703352:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949003.703356:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949003.703360:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949003.703366:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949003.703368:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 730 00000100:00000001:3.0:1784949003.703377:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949003.703379:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949003.703386:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558303872 : -115544151247744 : ffff96e9c7e8b480) 00000100:00000001:3.0:1784949003.703389:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949003.703393:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949003.703396:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f59e6b47 x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:664/0 lens 224/0 e 0 to 0 dl 1784949014 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949003.703400:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949003.703407:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949003.703419:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949003.703424:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000a400f603. 00000100:00000001:1.0:1784949003.703427:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949003.703430:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000a400f603. 00000100:00100000:1.0:1784949003.703439:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f59e6b47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289667840:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949003.703444:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289667840 00000020:00000001:1.0:1784949003.703447:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949003.703451:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949003.703453:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949003.703456:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949003.703458:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949003.703462:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949003.703466:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949003.703468:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949003.703470:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949003.703472:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949003.703474:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949003.703479:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949003.703481:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949003.703484:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000adca82ef. 02000000:00000001:1.0:1784949003.703487:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949003.703489:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949003.703491:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949003.703494:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289667840 00010000:00000001:1.0:1784949003.703497:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949003.703501:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f59e6b47 x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:664/0 lens 224/224 e 0 to 0 dl 1784949014 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949003.703510:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949003.703513:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949003.703515:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=20 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949003.703519:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949003.703522:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949003.703525:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949003.703528:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949003.703530:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949003.703533:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949003.703536:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949003.703539:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000100:00000200:1.0:1784949003.703542:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289667840, offset 224 00000400:00000200:1.0:1784949003.703548:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949003.703557:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949003.703563:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3503:3503:243:4294967295] 192.168.202.137@tcp LPNI seq info [2257:2257:8:4294967295] 00000400:00000200:1.0:1784949003.703575:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949003.703581:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949003.703585:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000800:00000200:1.0:1784949003.703591:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949003.703597:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949003.703601:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949003.703617:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949003.703620:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949003.703623:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949003.703624:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949003.703627:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949003.703631:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59e6b47 x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:664/0 lens 224/224 e 0 to 0 dl 1784949014 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949003.703643:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f59e6b47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289667840:12345-192.168.202.137@tcp:400: Request processed in 223us (620us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949003.703652:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 730 00000100:00000040:1.0:1784949003.703656:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949003.703659:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949003.703662:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949003.703666:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f2b75cde. 00000020:00000010:1.0:1784949003.703670:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949003.703675:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000052e1e01a. 00000020:00000040:1.0:1784949003.703682:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949003.703687:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949003.703737:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949003.703743:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000400:00000200:3.0:1784949003.703751:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949003.703755:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949003.703764:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949003.703771:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949003.703773:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000800:00000001:1.0:1784949003.703777:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784949003.703784:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000800:00000001:1.0:1784949003.703785:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.703797:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:1.0:1784949003.703803:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949003.703805:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949003.703827:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439900 00000400:00000200:1.0:1784949003.703841:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8afd [1] + 224 00000800:00000001:1.0:1784949003.703852:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949003.703861:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949003.703865:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949003.703873:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949003.703880:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949003.703886:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949003.703965:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949003.703987:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949003.703996:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949003.704001:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.704039:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949003.704044:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949003.704049:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949003.704051:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949003.704054:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000400:00000200:3.0:1784949003.704060:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949003.704062:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:3.0:1784949003.704065:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949003.704072:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949003.704083:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949003.704092:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949003.704094:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.704096:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949003.704098:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949003.704100:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949003.704102:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949003.704104:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949003.704109:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=20 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949003.704113:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=20 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949003.704115:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949003.704117:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949003.704119:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949003.704123:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949003.704132:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.704135:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949003.704138:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949003.704140:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949003.704142:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949003.704145:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949003.704153:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949003.704155:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949003.704157:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949003.704160:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949003.704168:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000d003ffea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289667840:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949003.704172:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.704175:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949003.704177:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000d003ffea x1871651289667840/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949003.704184:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949003.704186:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949003.704188:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:3.0:1784949003.704191:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949003.704193:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949003.704195:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949003.704197:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949003.704198:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b51ad697. 02000000:00000001:3.0:1784949003.704201:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949003.704203:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949003.704204:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000d003ffea. 00000100:00000001:3.0:1784949003.704210:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949003.704211:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.704213:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949003.704218:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949003.704219:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949003.704222:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949003.704224:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949003.745033:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305943084, at 4305943085 00000100:00000040:3.0:1784949003.745042:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305943084, at 4305943085 00000100:00000010:1.0:1784949003.745048:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00000100:00000040:2.0:1784949003.745050:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305943084, at 4305943085 00000100:00000010:3.0:1784949003.745059:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a2179890. 00000100:00000010:2.0:1784949003.745066:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000ef8f137f. 00000100:00000040:0.0:1784949003.745070:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305943084, at 4305943085 00000100:00000010:0.0:1784949003.745085:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e19b789f. 00010000:00000001:2.0:1784949004.532042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949004.532050:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949004.532058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949004.532060:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949004.532160:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949004.532167:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949004.532171:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949005.556040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949005.556050:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949005.556059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949005.556061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949005.556163:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949005.556169:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949005.556174:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949006.580034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949006.580043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949006.580051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949006.580053:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949006.580135:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949006.580141:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949006.580146:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949007.604033:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949007.604041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949007.604048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949007.604051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949007.604147:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949007.604154:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949007.604158:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949008.627981:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949008.627988:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949008.627994:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949008.627995:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949008.628056:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949008.628059:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949008.628062:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949008.820000:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949008.820014:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949008.820018:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949008.820020:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949008.820027:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949008.820029:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949008.820033:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1784949008.820039:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000487ebca5. 00000100:00000001:1.0:1784949008.820040:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820048:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949008.820049:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949008.820056:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949008.820060:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949008.820065:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949008.820074:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b51ad697. 00000100:00000001:0.0:1784949008.820081:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949008.820090:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949008.820103:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949008.820105:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949008.820110:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000487ebca5] to pc [ptlrpcd_00_03+3] req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949008.820144:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949008.820147:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949008.820151:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949008.820154:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949008.820158:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949008.820163:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000100:00000001:1.0:1784949008.820165:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784949008.820166:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949008.820173:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784949008.820173:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949008.820179:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949008.820180:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949008.820181:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949008.820182:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949008.820189:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949008.820190:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000eafb2cdc. 00000100:00000040:1.0:1784949008.820191:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949008.820198:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949008.820210:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949008.820213:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949008.820219:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949008.820245:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949008.820254:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949008.820255:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949008.820257:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949008.820269:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949008.820272:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_00+0] req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949008.820278:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000487ebca5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289667968:0@lo:400: 00000100:00000001:1.0:1784949008.820293:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949008.820302:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949008.820307:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949008.820313:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949008.820316:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949008.820324:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 00000100:00000040:0.0:1784949008.820326:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11285) 02000000:00000001:1.0:1784949008.820335:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949008.820347:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000011e3ad9. 00000400:00000010:1.0:1784949008.820361:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 00000100:00000001:3.0:1784949008.820363:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949008.820365:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949008.820370:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1784949008.820373:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949008.820374:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289667968, portal 25 00000100:00000001:3.0:1784949008.820377:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949008.820385:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949008.820390:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554977920 : -115544154573696 : ffff96e9c7b5f480) 00000100:00000001:3.0:1784949008.820390:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949008.820394:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949008.820398:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949008.820399:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949008.820401:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949008.820405:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949008.820414:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949008.820415:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949008.820420:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949008.820426:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000100:00000001:3.0:1784949008.820429:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:1.0:1784949008.820430:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289667968, offset 0 00000100:00000001:3.0:1784949008.820432:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949008.820437:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784949008.820440:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784949008.820440:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949008.820444:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949008.820448:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289668096:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949008.820454:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949008.820457:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1784949008.820459:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:3.0:1784949008.820460:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949008.820463:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949008.820469:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 02000000:00000001:3.0:1784949008.820472:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949008.820476:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439980 00000400:00000010:3.0:1784949008.820478:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000df521cee. 00000400:00000010:3.0:1784949008.820484:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000400:00000200:1.0:1784949008.820488:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x80e9 [2] + 0 00000400:00000010:1.0:1784949008.820495:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fe908aa6. 00000400:00000200:1.0:1784949008.820503:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d1142873 00000100:00000200:3.0:1784949008.820508:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668096, portal 10 00000100:00000001:3.0:1784949008.820512:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949008.820514:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949008.820515:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000400:00000200:1.0:1784949008.820523:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:3.0:1784949008.820525:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949008.820531:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d1142873 00000400:00000010:1.0:1784949008.820536:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d1142873. 00000100:00000001:3.0:1784949008.820537:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949008.820544:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949008.820546:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949008.820549:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:3.0:1784949008.820550:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000100:00000200:3.0:1784949008.820554:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289668096, offset 0 00000100:00000040:1.0:1784949008.820555:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000028574ea9 x1871651289667968 msgsize 224 00000100:00100000:1.0:1784949008.820564:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784949008.820565:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949008.820569:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949008.820580:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949008.820592:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949008.820593:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949008.820601:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949008.820608:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949008.820615:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949008.820623:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3504:3504:243:4294967295] 192.168.202.137@tcp LPNI seq info [2258:2258:8:4294967295] 00000100:00000001:2.0:1784949008.820633:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949008.820641:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289667968 02000000:00000001:2.0:1784949008.820664:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:3.0:1784949008.820665:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784949008.820667:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949008.820672:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949008.820678:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949008.820678:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949008.820681:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000100:00100000:2.0:1784949008.820684:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289667968 00000400:00000010:1.0:1784949008.820685:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000020:00000001:2.0:1784949008.820689:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000010:3.0:1784949008.820689:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000001:1.0:1784949008.820692:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:2.0:1784949008.820694:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784949008.820698:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1784949008.820699:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949008.820704:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000040:2.0:1784949008.820705:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784949008.820712:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:3.0:1784949008.820718:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949008.820724:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949008.820727:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949008.820732:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949008.820740:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820741:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820744:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1784949008.820746:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2132 usecs (3 jiffies) 00000100:00000001:3.0:1784949008.820751:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305948163 00000100:00000001:1.0:1784949008.820753:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820756:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305948202 00000100:00000010:3.0:1784949008.820758:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000006204bf6f. 00000020:00000001:2.0:1784949008.820782:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949008.820784:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949008.820790:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949008.820796:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305948163 00000100:00000001:1.0:1784949008.820800:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305948163 00000100:00000010:3.0:1784949008.820805:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:1.0:1784949008.820806:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305948202 00000100:00000001:3.0:1784949008.820812:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949008.820818:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305948163 00000100:00000001:1.0:1784949008.820820:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305948163 00000020:00000010:2.0:1784949008.820821:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000084bc9b0a. 00000100:00000010:3.0:1784949008.820825:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000001:2.0:1784949008.820831:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949008.820832:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305948202 00000100:00000001:3.0:1784949008.820833:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000010:2.0:1784949008.820857:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000084bc9b0a. 00000100:00000001:1.0:1784949008.820859:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305948163 00000100:00000001:3.0:1784949008.820859:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305948163 00000020:00000010:2.0:1784949008.820867:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000100:00000010:3.0:1784949008.820867:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000068c34307. 00000100:00000001:1.0:1784949008.820868:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305948202 00000100:00000001:3.0:1784949008.820876:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000010:2.0:1784949008.820878:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ce523b1b. 00000100:00000001:3.0:1784949008.820935:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949008.820936:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784949008.820938:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305948163 00000100:00000001:3.0:1784949008.820944:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820946:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949008.820948:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820950:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949008.820951:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949008.820953:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949008.820953:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949008.820958:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949008.820959:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949008.820962:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949008.820966:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820969:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949008.820973:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949008.820976:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.820978:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949008.820982:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949008.820985:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949008.820988:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949008.820990:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949008.821007:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949008.821011:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949008.821017:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 755 00000100:00000040:2.0:1784949008.821021:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949008.821024:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457133648 : -115544252417968 : ffff96e9c1e0f850) 00000100:00000040:2.0:1784949008.821030:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000028574ea9 x1871651289667968/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:669/0 lens 224/0 e 0 to 0 dl 1784949019 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949008.821040:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949008.821044:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000084bc9b0a. 00000100:00000001:2.0:1784949008.821047:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949008.821050:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000084bc9b0a. 00000800:00000200:0.0:1784949008.821058:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00100000:2.0:1784949008.821058:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000028574ea9 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289667968:12345-0@lo:400: 00000800:00000001:1.0:1784949008.821062:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949008.821066:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000200:2.0:1784949008.821069:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289667968 00000020:00000001:2.0:1784949008.821074:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784949008.821075:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949008.821078:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000001:1.0:1784949008.821080:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949008.821082:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949008.821086:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949008.821091:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949008.821091:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:0.0:1784949008.821093:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000020:00000001:2.0:1784949008.821096:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000010:0.0:1784949008.821098:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000800:00000001:1.0:1784949008.821101:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949008.821101:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949008.821104:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1784949008.821105:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949008.821110:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:0.0:1784949008.821115:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949008.821116:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:1.0:1784949008.821121:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:2.0:1784949008.821122:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949008.821129:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949008.821150:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949008.821157:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949008.821160:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1784949008.821166:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d78419ed. 00000400:00000200:1.0:1784949008.821167:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439a00 00000100:00000040:0.0:1784949008.821171:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949008.821174:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949008.821182:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949008.821190:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949008.821197:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 164784 00010000:00000040:2.0:1784949008.821198:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289667968 00000100:00000001:0.0:1784949008.821203:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949008.821208:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949008.821213:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000200:2.0:1784949008.821220:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000028574ea9 x1871651289667968/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:669/0 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949008.821221:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784949008.821236:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949008.821240:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949008.821243:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949008.821245:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=30 v=5 (1 1 1 1) 00000400:00000200:1.0:1784949008.821252:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949008.821255:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949008.821264:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949008.821270:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:1.0:1784949008.821271:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:2.0:1784949008.821275:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949008.821281:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949008.821285:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949008.821288:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:2.0:1784949008.821289:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000200:1.0:1784949008.821296:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000010:2.0:1784949008.821298:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000100:00000200:2.0:1784949008.821309:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289667968, offset 224 00000100:00000010:1.0:1784949008.821311:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ddd830de. 00000400:00000200:2.0:1784949008.821320:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:1.0:1784949008.821321:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000ddd830de x1871651289668096 msgsize 224 00000400:00000200:2.0:1784949008.821339:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:1.0:1784949008.821349:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784949008.821359:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439980 00000400:00000200:2.0:1784949008.821367:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8b11 [1] + 224 00000100:00000001:1.0:1784949008.821370:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949008.821376:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949008.821383:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:1.0:1784949008.821385:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949008.821390:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:1.0:1784949008.821396:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949008.821399:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949008.821415:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949008.821426:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949008.821432:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668096 02000000:00000001:1.0:1784949008.821437:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949008.821437:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949008.821445:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949008.821448:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949008.821450:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 02000000:00000001:1.0:1784949008.821454:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949008.821456:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00100000:1.0:1784949008.821465:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668096 00000100:00000001:2.0:1784949008.821468:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:1.0:1784949008.821474:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949008.821474:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000040:1.0:1784949008.821482:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:2.0:1784949008.821486:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949008.821488:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949008.821496:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:2.0:1784949008.821497:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:1.0:1784949008.821507:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:2.0:1784949008.821508:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949008.821513:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949008.821517:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949008.821519:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949008.821527:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000028574ea9 x1871651289667968/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:669/0 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949008.821530:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949008.821540:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e374d605. 00000100:00100000:2.0:1784949008.821548:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000028574ea9 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289667968:12345-0@lo:400: Request processed in 501us (986us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949008.821551:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949008.821563:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e374d605. 00000100:00100000:2.0:1784949008.821572:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 755 00000100:00000040:2.0:1784949008.821577:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000010:1.0:1784949008.821580:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000072ce5ef0. 00000100:00000001:2.0:1784949008.821581:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949008.821586:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949008.821591:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f2b75cde. 00000020:00000010:2.0:1784949008.821593:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ce523b1b. 00000020:00000010:1.0:1784949008.821600:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000020:00000010:2.0:1784949008.821602:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949008.821610:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 00000100:00000040:1.0:1784949008.821618:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000040:2.0:1784949008.821621:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949008.821630:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:2.0:1784949008.821631:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949008.821634:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000010:2.0:1784949008.821636:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d78419ed. 00000100:00000001:1.0:1784949008.821640:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949008.821642:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949008.821647:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949008.821666:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:2.0:1784949008.821670:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949008.821671:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949008.821675:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949008.821679:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ac0969b3. 00000400:00000010:2.0:1784949008.821684:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000aad879a6. 00000100:00000001:1.0:1784949008.821691:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949008.821704:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949008.821706:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949008.821712:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949008.821715:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 731 00000100:00000001:2.0:1784949008.821717:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784949008.821720:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000001:2.0:1784949008.821724:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784949008.821725:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549156224 : -115544160395392 : ffff96e9c75d1f80) 00000400:00000001:2.0:1784949008.821729:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949008.821733:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000011e3ad9. 00000100:00000040:1.0:1784949008.821735:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ddd830de x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:669/0 lens 224/0 e 0 to 0 dl 1784949019 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949008.821738:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0c9b918 00000400:00000010:2.0:1784949008.821743:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f0c9b918. 00000100:00000001:2.0:1784949008.821748:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784949008.821756:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:2.0:1784949008.821756:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949008.821762:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e374d605. 00000100:00000001:1.0:1784949008.821767:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949008.821774:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e374d605. 00000100:00000200:2.0:1784949008.821776:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949008.821785:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ddd830de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289668096:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949008.821797:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668096 00000100:00000001:2.0:1784949008.821798:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1784949008.821803:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949008.821804:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949008.821809:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:1.0:1784949008.821810:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949008.821813:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949008.821813:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949008.821816:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:1.0:1784949008.821818:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:2.0:1784949008.821820:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949008.821821:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 02000000:00000001:2.0:1784949008.821826:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949008.821827:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949008.821832:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949008.821834:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=30 v=5 (5 5 5 5) 00000020:00000001:1.0:1784949008.821835:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949008.821838:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949008.821842:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949008.821843:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=30 v=5 (1 1 1 1) 00000020:00000001:1.0:1784949008.821846:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949008.821849:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949008.821852:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.821853:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00010000:00000001:2.0:1784949008.821856:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000001:1.0:1784949008.821857:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00010000:00000001:2.0:1784949008.821860:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949008.821863:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000003fc6890. 00000100:00000001:2.0:1784949008.821865:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949008.821868:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.821872:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949008.821872:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949008.821900:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949008.821903:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000040:1.0:1784949008.821933:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668096 00000100:00000001:2.0:1784949008.821935:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00010000:00000001:1.0:1784949008.821939:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949008.821940:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:1.0:1784949008.821947:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ddd830de x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:669/0 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949008.821948:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949008.821976:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949008.821981:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949008.821985:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=25 v=5 (1 1 1 1) 00000100:00100000:2.0:1784949008.821985:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000487ebca5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289667968:0@lo:400: 00000100:00000001:1.0:1784949008.821992:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949008.821999:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949008.822001:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949008.822002:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949008.822010:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784949008.822010:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784949008.822014:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.822018:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949008.822020:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000487ebca5 x1871651289667968/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949008.822024:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949008.822029:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000100:00000200:1.0:1784949008.822035:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289668096, offset 224 00000400:00000200:1.0:1784949008.822045:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949008.822047:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949008.822053:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:1.0:1784949008.822057:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000010:2.0:1784949008.822057:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 00000400:00000200:1.0:1784949008.822066:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3505:3505:243:4294967295] 192.168.202.137@tcp LPNI seq info [2259:2259:8:4294967295] 02000000:00000001:2.0:1784949008.822068:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949008.822073:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949008.822077:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949008.822083:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:1.0:1784949008.822085:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000010:2.0:1784949008.822087:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b51ad697. 02000000:00000001:2.0:1784949008.822093:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:1.0:1784949008.822094:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784949008.822097:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000010:1.0:1784949008.822100:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000100:00000010:2.0:1784949008.822100:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000487ebca5. 00000800:00000200:1.0:1784949008.822111:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949008.822114:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949008.822117:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949008.822122:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:2.0:1784949008.822123:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949008.822130:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949008.822132:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949008.822136:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949008.822139:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949008.822141:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949008.822146:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949008.822152:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949008.822154:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949008.822156:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949008.822158:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949008.822162:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ddd830de x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:669/0 lens 224/224 e 0 to 0 dl 1784949019 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949008.822173:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ddd830de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289668096:12345-192.168.202.137@tcp:400: Request processed in 416us (849us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949008.822180:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 731 00000100:00000040:1.0:1784949008.822182:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949008.822185:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949008.822187:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949008.822190:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f2b75cde. 00000020:00000010:1.0:1784949008.822194:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949008.822197:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000072ce5ef0. 00000020:00000040:1.0:1784949008.822201:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949008.822205:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949008.822227:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949008.822232:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000400:00000200:0.0:1784949008.822238:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949008.822244:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949008.822248:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000400:00000010:0.0:1784949008.822250:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000100:00000001:0.0:1784949008.822254:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949008.822255:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949008.822262:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949008.822268:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949008.822271:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949008.822275:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949008.822280:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949008.822288:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439a00 00000400:00000200:0.0:1784949008.822293:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8b19 [1] + 224 00000800:00000001:0.0:1784949008.822298:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949008.822301:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949008.822303:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949008.822305:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949008.822308:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949008.822311:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949008.822319:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949008.822335:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949008.822339:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949008.822342:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949008.822374:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949008.822378:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949008.822382:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949008.822383:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949008.822385:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000400:00000200:3.0:1784949008.822389:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949008.822391:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:3.0:1784949008.822393:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949008.822398:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949008.822406:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949008.822413:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949008.822414:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949008.822416:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949008.822417:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949008.822418:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949008.822420:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949008.822422:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949008.822426:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=25 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949008.822429:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=25 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949008.822431:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949008.822432:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949008.822434:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949008.822437:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949008.822448:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949008.822451:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949008.822453:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949008.822455:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949008.822456:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949008.822458:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949008.822463:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949008.822465:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949008.822466:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949008.822468:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949008.822474:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289668096:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949008.822477:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949008.822479:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949008.822481:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289668096/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949008.822487:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949008.822488:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949008.822490:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:3.0:1784949008.822492:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949008.822493:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949008.822494:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949008.822496:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949008.822497:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000eafb2cdc. 02000000:00000001:3.0:1784949008.822500:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949008.822501:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949008.822502:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:3.0:1784949008.822511:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949008.822512:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949008.822513:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949008.822517:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949008.822519:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949008.822520:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949008.822522:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949008.863009:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305948202, at 4305948203 00000100:00000040:1.0:1784949008.863019:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305948202, at 4305948203 00000100:00000010:2.0:1784949008.863022:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000010:1.0:1784949008.863034:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:0.0:1784949008.863037:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305948202, at 4305948203 00000100:00000010:0.0:1784949008.863050:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000006204bf6f. 00000100:00000040:3.0:1784949008.864032:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305948202, at 4305948204 00000100:00000010:3.0:1784949008.864042:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000068c34307. 00010000:00000001:2.0:1784949009.652019:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949009.652028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949009.652035:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949009.652037:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949009.652138:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949009.652143:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949009.652147:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949009.844009:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949009.844024:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949009.844026:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949009.844032:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949009.844036:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949009.844040:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949009.844044:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949010.675985:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949010.675992:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949010.675998:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949010.675999:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949010.676063:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949010.676068:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949010.676071:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949010.867961:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949010.867974:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949010.867976:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949010.867980:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949010.867982:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949010.867985:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949010.867988:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949011.700001:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949011.700008:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949011.700014:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949011.700017:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949011.700019:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949011.700022:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949011.700024:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949011.700026:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949011.700028:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949011.700029:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949011.700126:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949011.700131:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949011.700133:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949011.892003:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949011.892016:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949011.892018:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949011.892022:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949011.892025:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949011.892029:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949011.892031:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949012.724045:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949012.724057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949012.724068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949012.724071:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949012.724154:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949012.724162:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949012.724167:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949012.916078:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949012.916093:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949012.916096:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949012.916103:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949012.916107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949012.916111:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949012.916114:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949013.747995:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949013.748004:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949013.748010:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949013.748012:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949013.748111:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949013.748118:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949013.748124:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949013.940008:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949013.940024:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949013.940028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949013.940034:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949013.940038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949013.940044:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949013.940046:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949013.940048:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949013.940060:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949013.940072:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000033c07e20. 00000020:00000040:0.0:1784949013.940080:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949013.940087:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949013.940091:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949013.940098:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949013.940109:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002aeee8e9. 00000100:00000001:0.0:1784949013.940120:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949013.940133:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949013.940149:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949013.940154:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949013.940163:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033c07e20] to pc [ptlrpcd_00_01+1] req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949013.940205:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949013.940210:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949013.940216:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949013.940222:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949013.940227:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949013.940234:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000100:00000001:1.0:1784949013.940234:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784949013.940244:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949013.940244:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949013.940246:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1784949013.940255:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949013.940256:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784949013.940258:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949013.940262:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949013.940270:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949013.940276:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:0.0:1784949013.940280:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000006487a147. 00000100:00000001:1.0:1784949013.940283:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949013.940289:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949013.940294:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949013.940301:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949013.940302:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949013.940310:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949013.940320:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949013.940323:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949013.940326:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949013.940335:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_02+2] req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949013.940362:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949013.940367:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949013.940379:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949013.940384:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949013.940389:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949013.940395:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000033c07e20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289668224:0@lo:400: 00000100:00000001:1.0:1784949013.940402:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949013.940412:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949013.940412:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949013.940418:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949013.940424:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784949013.940428:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11290) 02000000:00000010:1.0:1784949013.940432:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000072ce5ef0. 00000100:00000001:2.0:1784949013.940435:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949013.940439:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.940443:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949013.940449:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:1.0:1784949013.940450:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000011e3ad9. 00000100:00000040:2.0:1784949013.940460:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949013.940462:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000200:1.0:1784949013.940478:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668224, portal 25 00000100:00000001:2.0:1784949013.940480:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949013.940484:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949013.940486:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949013.940490:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554967168 : -115544154584448 : ffff96e9c7b5ca80) 02000000:00000001:2.0:1784949013.940494:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949013.940500:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949013.940503:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949013.940509:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949013.940525:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289668352:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949013.940529:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949013.940536:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949013.940537:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949013.940539:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949013.940547:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949013.940548:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:1.0:1784949013.940549:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 02000000:00000001:2.0:1784949013.940555:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949013.940562:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289668224, offset 0 02000000:00000001:2.0:1784949013.940563:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949013.940572:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007a6587e5. 02000000:00000001:2.0:1784949013.940578:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949013.940580:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:2.0:1784949013.940591:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f4e53d4f. 00000400:00000010:2.0:1784949013.940602:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000200:1.0:1784949013.940603:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:2.0:1784949013.940614:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668352, portal 10 00000100:00000001:2.0:1784949013.940619:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949013.940623:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000400:00000200:1.0:1784949013.940630:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439a80 00000400:00000200:1.0:1784949013.940641:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8185 [2] + 0 00000100:00000040:2.0:1784949013.940648:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949013.940650:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a31fc1e7. 00000400:00000200:1.0:1784949013.940659:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000001d1e90a4 00000400:00000200:1.0:1784949013.940673:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949013.940675:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949013.940684:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949013.940691:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000200:1.0:1784949013.940698:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:2.0:1784949013.940704:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289668352, offset 0 00000400:00000200:1.0:1784949013.940708:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001d1e90a4 00000400:00000010:1.0:1784949013.940715:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000001d1e90a4. 00000100:00000001:1.0:1784949013.940722:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949013.940729:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1784949013.940731:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949013.940741:0:337119:0:(events.c:337:request_in_callback()) incoming req@000000009b82ab0d x1871651289668224 msgsize 224 00000100:00100000:1.0:1784949013.940757:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:2.0:1784949013.940762:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:1.0:1784949013.940765:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949013.940780:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949013.940788:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949013.940795:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949013.940798:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949013.940806:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949013.940811:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3506:3506:243:4294967295] 192.168.202.137@tcp LPNI seq info [2260:2260:8:4294967295] 00000400:00000200:2.0:1784949013.940829:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949013.940864:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949013.940870:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000800:00000010:2.0:1784949013.940871:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005c3de731. 00000400:00000010:1.0:1784949013.940890:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000800:00000200:2.0:1784949013.940900:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949013.940903:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949013.940952:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949013.940968:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949013.940969:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:2.0:1784949013.940978:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005c3de731 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:1.0:1784949013.940982:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949013.940995:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949013.940999:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949013.941004:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949013.941011:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949013.941012:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949013.941014:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2132 usecs (3 jiffies) 00000100:00100000:0.0:1784949013.941020:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668224 00000100:00000001:1.0:1784949013.941027:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305953323 00000100:00000001:2.0:1784949013.941028:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305953284 02000000:00000001:0.0:1784949013.941029:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949013.941036:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949013.941042:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949013.941045:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 02000000:00000001:0.0:1784949013.941048:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949013.941056:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668224 00000100:00000001:1.0:1784949013.941059:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1784949013.941060:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949013.941068:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949013.941072:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949013.941073:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305953323 00000100:00000001:2.0:1784949013.941075:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305953323 00000020:00000040:0.0:1784949013.941077:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:1.0:1784949013.941085:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:2.0:1784949013.941086:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305953284 00000020:00000001:0.0:1784949013.941087:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949013.941096:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1784949013.941100:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949013.941105:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305953323 00000100:00000001:2.0:1784949013.941106:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305953323 00000100:00000001:0.0:1784949013.941109:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949013.941120:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305953284 00000100:00000010:1.0:1784949013.941123:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000020:00000010:0.0:1784949013.941124:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784949013.941134:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949013.941136:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:0.0:1784949013.941140:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:1.0:1784949013.941147:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305953323 00000100:00000001:2.0:1784949013.941148:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305953323 00000020:00000010:0.0:1784949013.941151:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:2.0:1784949013.941160:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305953284 00000100:00000010:1.0:1784949013.941162:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000010:0.0:1784949013.941166:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007035a559. 00000100:00000001:1.0:1784949013.941175:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000010:0.0:1784949013.941178:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949013.941182:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949013.941183:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305953323 00000100:00000040:0.0:1784949013.941192:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949013.941195:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.941196:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949013.941200:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949013.941208:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949013.941209:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949013.941212:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949013.941216:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.941218:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949013.941226:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.941227:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949013.941233:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.941242:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949013.941247:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949013.941253:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949013.941254:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949013.941260:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949013.941265:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.941270:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949013.941282:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949013.941286:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949013.941288:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949013.941296:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949013.941303:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 756 00000100:00000040:0.0:1784949013.941309:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949013.941311:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949013.941314:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457129552 : -115544252422064 : ffff96e9c1e0e850) 00000100:00000001:1.0:1784949013.941317:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949013.941326:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009b82ab0d x1871651289668224/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:674/0 lens 224/0 e 0 to 0 dl 1784949024 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949013.941340:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949013.941344:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784949013.941349:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949013.941354:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00100000:0.0:1784949013.941366:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009b82ab0d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289668224:12345-0@lo:400: 00000100:00000200:0.0:1784949013.941374:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668224 00000020:00000001:0.0:1784949013.941381:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949013.941389:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949013.941394:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949013.941399:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949013.941404:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949013.941411:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949013.941416:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949013.941422:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949013.941427:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949013.941433:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949013.941439:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949013.941444:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949013.941450:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949013.941459:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006e175381. 02000000:00000001:0.0:1784949013.941465:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949013.941470:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949013.941474:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949013.941478:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668224 00010000:00000001:0.0:1784949013.941483:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949013.941491:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009b82ab0d x1871651289668224/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:674/0 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949013.941502:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949013.941506:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949013.941511:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=35 v=5 (1 1 1 1) 00000800:00000200:3.0:1784949013.941513:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949013.941519:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1784949013.941522:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005c3de731. 00000100:00000040:0.0:1784949013.941530:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949013.941537:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:3.0:1784949013.941538:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949013.941547:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:0.0:1784949013.941550:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949013.941558:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949013.941559:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949013.941570:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:2.0:1784949013.941570:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949013.941572:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:3.0:1784949013.941580:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000100:00000040:0.0:1784949013.941582:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:2.0:1784949013.941583:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949013.941588:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:0.0:1784949013.941595:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000800:00000001:2.0:1784949013.941596:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949013.941599:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949013.941609:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289668224, offset 224 00000400:00000200:2.0:1784949013.941622:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949013.941634:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949013.941641:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949013.941652:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949013.941656:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439b00 00000400:00000200:0.0:1784949013.941660:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949013.941683:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 165008 00000400:00000200:0.0:1784949013.941691:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439a80 00000800:00000001:2.0:1784949013.941709:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949013.941710:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949013.941716:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8b2d [1] + 224 00000100:00000001:3.0:1784949013.941722:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1784949013.941724:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949013.941729:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949013.941732:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949013.941741:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784949013.941749:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949013.941754:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949013.941759:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949013.941764:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949013.941766:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:2.0:1784949013.941777:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b3c18d2b. 00000100:00000040:2.0:1784949013.941785:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000b3c18d2b x1871651289668352 msgsize 224 00000100:00000040:0.0:1784949013.941792:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949013.941803:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784949013.941829:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949013.941864:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949013.941865:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949013.941876:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949013.941886:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:0.0:1784949013.941894:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000100:00000001:0.0:1784949013.941902:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949013.941956:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784949013.941960:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949013.941961:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784949013.941968:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949013.941968:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668352 00000100:00000001:1.0:1784949013.941970:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949013.941978:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949013.941983:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:0.0:1784949013.941985:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:1.0:1784949013.941986:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949013.941988:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949013.941993:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1784949013.941998:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949013.942001:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:1.0:1784949013.942001:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000011e3ad9. 00000100:00100000:3.0:1784949013.942005:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668352 00000020:00000001:0.0:1784949013.942010:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949013.942013:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1784949013.942014:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000020:00000040:3.0:1784949013.942020:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:1.0:1784949013.942022:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000020:00000001:0.0:1784949013.942024:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:3.0:1784949013.942027:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949013.942033:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:3.0:1784949013.942035:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:0.0:1784949013.942041:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009b82ab0d x1871651289668224/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:674/0 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949013.942044:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784949013.942045:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949013.942056:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949013.942068:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949013.942077:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000001:3.0:1784949013.942087:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949013.942097:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000020:00000010:3.0:1784949013.942111:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000015713dee. 00000100:00000200:1.0:1784949013.942117:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949013.942120:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009b82ab0d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289668224:12345-0@lo:400: Request processed in 770us (1370us total) trans 0 rc 0/0 00000020:00000010:3.0:1784949013.942123:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000fe2862da. 00000020:00000010:3.0:1784949013.942134:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949013.942154:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:0.0:1784949013.942157:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 756 00000100:00000040:3.0:1784949013.942158:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949013.942159:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949013.942169:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:0.0:1784949013.942172:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949013.942173:0:337119:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:3.0:1784949013.942174:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:1.0:1784949013.942182:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949013.942183:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949013.942184:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949013.942186:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949013.942193:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:1.0:1784949013.942196:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949013.942198:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000020:00000010:0.0:1784949013.942202:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007035a559. 00000100:00000001:3.0:1784949013.942206:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:1.0:1784949013.942209:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949013.942211:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949013.942212:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00001000:1.0:1784949013.942223:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=35 v=5 (5 5 5 5) 00000020:00000010:0.0:1784949013.942225:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000020:00000040:0.0:1784949013.942236:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:1.0:1784949013.942238:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=35 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949013.942239:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949013.942245:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784949013.942252:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006e175381. 00000100:00000001:1.0:1784949013.942256:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000001:0.0:1784949013.942259:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949013.942264:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949013.942265:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949013.942266:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949013.942272:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000001:1.0:1784949013.942279:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:0.0:1784949013.942283:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:1.0:1784949013.942290:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949013.942291:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 732 00000400:00000010:0.0:1784949013.942298:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b758fcf6. 00000100:00000001:2.0:1784949013.942298:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949013.942304:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949013.942304:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949013.942306:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:0.0:1784949013.942310:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e65b0b8e. 00000100:00000001:2.0:1784949013.942315:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949013.942316:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529558295808 : -115544151255808 : ffff96e9c7e89500) 00000100:00000040:1.0:1784949013.942324:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949013.942327:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949013.942336:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b3c18d2b x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:674/0 lens 224/0 e 0 to 0 dl 1784949024 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949013.942364:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949013.942374:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949013.942375:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949013.942381:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949013.942383:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000040:1.0:1784949013.942390:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949013.942392:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949013.942403:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000100:00100000:3.0:1784949013.942419:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b3c18d2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289668352:12345-192.168.202.137@tcp:400: 00000100:00100000:1.0:1784949013.942422:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000033c07e20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289668224:0@lo:400: 00000100:00000200:3.0:1784949013.942436:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668352 00000100:00000001:1.0:1784949013.942440:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949013.942441:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949013.942449:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1784949013.942451:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949013.942456:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949013.942457:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000033c07e20 x1871651289668224/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949013.942465:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949013.942471:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949013.942482:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949013.942484:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1784949013.942487:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949013.942490:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1784949013.942494:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000010:1.0:1784949013.942496:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000072ce5ef0. 00000020:00000001:3.0:1784949013.942500:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:1.0:1784949013.942505:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949013.942506:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949013.942512:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:3.0:1784949013.942513:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000040:1.0:1784949013.942519:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949013.942522:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:1.0:1784949013.942526:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:3.0:1784949013.942528:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949013.942532:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002aeee8e9. 02000000:00000010:3.0:1784949013.942535:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009b7d615c. 02000000:00000001:1.0:1784949013.942541:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949013.942542:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949013.942547:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1784949013.942549:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949013.942554:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000033c07e20. 00000020:00000001:3.0:1784949013.942555:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949013.942565:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668352 00010000:00000001:3.0:1784949013.942568:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949013.942573:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00010000:00000200:3.0:1784949013.942575:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b3c18d2b x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:674/0 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949013.942579:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949013.942588:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949013.942599:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:3.0:1784949013.942601:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949013.942603:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949013.942607:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949013.942610:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00001000:3.0:1784949013.942613:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=30 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949013.942616:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949013.942623:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949013.942629:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949013.942632:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949013.942636:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949013.942639:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949013.942644:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949013.942648:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949013.942652:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000200:3.0:1784949013.942656:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289668352, offset 224 00000400:00000200:3.0:1784949013.942663:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949013.942673:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949013.942680:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3507:3507:243:4294967295] 192.168.202.137@tcp LPNI seq info [2261:2261:8:4294967295] 00000400:00000200:3.0:1784949013.942696:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949013.942706:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949013.942712:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949013.942720:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949013.942729:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949013.942735:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949013.942759:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949013.942765:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949013.942770:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949013.942773:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949013.942777:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949013.942784:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b3c18d2b x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:674/0 lens 224/224 e 0 to 0 dl 1784949024 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949013.942801:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b3c18d2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289668352:12345-192.168.202.137@tcp:400: Request processed in 427us (1013us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949013.942812:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 732 00000100:00000040:3.0:1784949013.942815:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949013.942818:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949013.942820:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949013.942824:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000fe2862da. 00000020:00000010:3.0:1784949013.942829:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949013.942833:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000015713dee. 00000020:00000040:3.0:1784949013.942863:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949013.942869:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949013.943031:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949013.943038:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:0.0:1784949013.943046:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949013.943051:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949013.943058:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949013.943066:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000800:00000001:3.0:1784949013.943067:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949013.943071:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000800:00000001:3.0:1784949013.943073:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949013.943083:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784949013.943084:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949013.943090:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949013.943099:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949013.943114:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439b00 00000400:00000200:3.0:1784949013.943122:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8b35 [1] + 224 00000800:00000001:3.0:1784949013.943129:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949013.943137:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949013.943140:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949013.943145:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949013.943151:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949013.943155:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949013.943166:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949013.943188:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949013.943195:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949013.943199:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949013.943290:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949013.943297:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949013.943303:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949013.943305:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949013.943308:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f4e53d4f. 00000400:00000200:2.0:1784949013.943315:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784949013.943317:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949013.943321:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949013.943329:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949013.943342:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949013.943360:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949013.943362:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949013.943365:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949013.943367:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949013.943369:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949013.943371:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949013.943374:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949013.943380:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=30 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949013.943384:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=30 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949013.943387:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949013.943388:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949013.943391:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949013.943395:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949013.943404:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.943407:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949013.943409:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949013.943411:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949013.943413:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949013.943416:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949013.943424:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949013.943427:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949013.943429:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949013.943432:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949013.943441:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289668352:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949013.943446:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949013.943448:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949013.943450:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289668352/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949029 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949013.943465:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949013.943467:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949013.943470:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007a6587e5. 02000000:00000001:2.0:1784949013.943476:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949013.943478:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949013.943479:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949013.943482:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949013.943484:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000006487a147. 02000000:00000001:2.0:1784949013.943487:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949013.943489:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949013.943490:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949013.943501:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949013.943503:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949013.943505:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949013.943510:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949013.943512:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949013.943514:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949013.943516:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949013.984027:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305953323, at 4305953324 00000100:00000040:1.0:1784949013.984038:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305953323, at 4305953324 00000100:00000010:3.0:1784949013.984043:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:2.0:1784949013.984048:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305953323, at 4305953324 00000100:00000010:1.0:1784949013.984055:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:2.0:1784949013.984069:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:0.0:1784949013.984090:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305953323, at 4305953324 00000100:00000010:0.0:1784949013.984104:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949014.772036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949014.772046:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949014.772053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949014.772055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949014.772140:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949014.772146:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949014.772151:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949014.964010:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949014.964026:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949014.964029:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949014.964036:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949014.964039:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949014.964044:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949014.964047:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949015.796044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949015.796053:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949015.796061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949015.796063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949015.796182:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949015.796190:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949015.796197:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949015.988043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949015.988061:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949015.988064:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949015.988070:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949015.988074:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949015.988078:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949015.988082:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949016.820028:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949016.820039:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949016.820047:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949016.820049:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949016.820171:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949016.820177:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949016.820181:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949017.012004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949017.012021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949017.012024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949017.012031:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949017.012036:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949017.012041:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949017.012045:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949017.844031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949017.844040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949017.844048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949017.844050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949017.844130:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949017.844136:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949017.844141:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949018.036041:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949018.036056:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949018.036058:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949018.036065:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949018.036068:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949018.036074:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949018.036077:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949018.868027:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949018.868036:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949018.868045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949018.868047:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949018.868135:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949018.868144:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949018.868150:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949019.060008:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949019.060025:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949019.060028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949019.060034:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949019.060038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949019.060042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060046:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949019.060050:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949019.060058:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949019.060067:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b16a92e. 00000020:00000040:2.0:1784949019.060073:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949019.060078:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949019.060081:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949019.060086:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949019.060094:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fd30a9ad. 00000100:00000001:2.0:1784949019.060100:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949019.060109:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949019.060119:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949019.060122:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949019.060127:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b16a92e] to pc [ptlrpcd_00_03+3] req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949019.060156:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949019.060159:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949019.060162:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949019.060165:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949019.060168:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949019.060183:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949019.060185:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949019.060188:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949019.060189:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060191:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:2.0:1784949019.060193:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000067c965f9. 00000020:00000040:2.0:1784949019.060198:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000040:0.0:1784949019.060199:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949019.060203:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949019.060206:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949019.060209:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949019.060216:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fbc83221. 00000100:00000001:0.0:1784949019.060223:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949019.060224:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060229:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949019.060234:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1784949019.060238:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949019.060243:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949019.060244:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949019.060248:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949019.060249:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949019.060254:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067c965f9] to pc [ptlrpcd_00_00+0] req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949019.060255:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289668480:0@lo:400: 00000100:00000001:0.0:1784949019.060271:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949019.060279:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949019.060282:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949019.060287:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949019.060291:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949019.060296:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 00000100:00000040:2.0:1784949019.060298:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11296) 02000000:00000001:0.0:1784949019.060301:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949019.060311:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000b3b2e3c. 00000100:00000001:3.0:1784949019.060316:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784949019.060320:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00100000:3.0:1784949019.060323:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:3.0:1784949019.060327:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949019.060328:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949019.060330:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668480, portal 25 00000100:00000001:2.0:1784949019.060332:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949019.060334:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949019.060336:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949019.060338:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949019.060341:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482521600 : -115544227030016 : ffff96e9c3645c00) 00000100:00000001:3.0:1784949019.060343:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949019.060347:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949019.060350:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949019.060354:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949019.060360:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949019.060386:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949019.060387:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949019.060390:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949019.060395:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784949019.060396:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949019.060400:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784949019.060402:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 02000000:00000001:3.0:1784949019.060404:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949019.060410:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289668480, offset 0 00000100:00100000:3.0:1784949019.060411:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000067c965f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289668608:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949019.060420:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949019.060422:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949019.060424:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949019.060428:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784949019.060428:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949019.060434:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000015713dee. 02000000:00000001:3.0:1784949019.060438:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949019.060443:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000df521cee. 00000400:00000200:0.0:1784949019.060449:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:3.0:1784949019.060449:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000100:00000200:3.0:1784949019.060456:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668608, portal 10 00000100:00000001:3.0:1784949019.060459:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949019.060460:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562021248 : -115544147530368 : ffff96e9c8216d80) 00000100:00000040:3.0:1784949019.060466:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949019.060470:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439b80 00000100:00000001:3.0:1784949019.060481:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949019.060491:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x81a1 [2] + 0 00000100:00000040:3.0:1784949019.060491:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949019.060497:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000400:00000010:0.0:1784949019.060501:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b5489cc4. 00000400:00000200:0.0:1784949019.060508:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000fc35f5fb 00000100:00000200:3.0:1784949019.060512:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289668608, offset 0 00000400:00000200:0.0:1784949019.060517:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949019.060523:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949019.060528:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949019.060544:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fc35f5fb 00000400:00000200:3.0:1784949019.060547:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000010:0.0:1784949019.060549:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000fc35f5fb. 00000400:00000200:3.0:1784949019.060556:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784949019.060558:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:3.0:1784949019.060562:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000200:0.0:1784949019.060564:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1784949019.060569:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1784949019.060572:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000068fd7440 x1871651289668480 msgsize 224 00000400:00000200:3.0:1784949019.060574:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00100000:0.0:1784949019.060582:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949019.060587:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949019.060589:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3508:3508:243:4294967295] 192.168.202.137@tcp LPNI seq info [2262:2262:8:4294967295] 00000100:00000001:0.0:1784949019.060602:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949019.060611:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949019.060620:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949019.060626:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000800:00000010:3.0:1784949019.060628:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000400:00000010:0.0:1784949019.060632:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000001:0.0:1784949019.060639:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784949019.060644:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784949019.060647:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949019.060660:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949019.060667:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949019.060682:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949019.060687:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:0.0:1784949019.060688:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949019.060688:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949019.060693:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668480 00000100:00000040:3.0:1784949019.060697:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2135 usecs (3 jiffies) 02000000:00000001:2.0:1784949019.060699:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949019.060704:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949019.060706:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305958403 00000100:00000001:2.0:1784949019.060709:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060713:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949019.060716:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949019.060716:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000068c34307. 00000100:00000001:0.0:1784949019.060718:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784949019.060725:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668480 00000100:00000001:0.0:1784949019.060729:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949019.060729:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949019.060729:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000040:2.0:1784949019.060733:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784949019.060738:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949019.060740:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305958442 00000100:00000001:3.0:1784949019.060740:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305958403 00000020:00000040:2.0:1784949019.060742:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784949019.060749:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784949019.060749:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077930f7d. 00000100:00000001:0.0:1784949019.060753:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305958403 00000020:00000001:2.0:1784949019.060756:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949019.060757:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949019.060763:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305958442 00000100:00000001:2.0:1784949019.060764:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949019.060765:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305958403 00000020:00000010:2.0:1784949019.060770:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000100:00000001:0.0:1784949019.060773:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305958403 00000100:00000010:3.0:1784949019.060774:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00000001:2.0:1784949019.060775:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949019.060781:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000010:2.0:1784949019.060783:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000100:00000001:0.0:1784949019.060787:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305958442 00000100:00000001:3.0:1784949019.060788:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305958403 00000020:00000010:2.0:1784949019.060792:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000100:00000001:0.0:1784949019.060817:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305958403 00000020:00000010:2.0:1784949019.060819:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ce523b1b. 00000100:00000001:0.0:1784949019.060827:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305958442 00000020:00000010:2.0:1784949019.060827:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000010:3.0:1784949019.060827:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007024d30a. 00000100:00000001:3.0:1784949019.060836:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000040:2.0:1784949019.060844:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949019.060844:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949019.060847:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305958403 00000100:00000001:3.0:1784949019.060850:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949019.060851:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949019.060856:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949019.060856:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949019.060857:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949019.060862:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060865:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949019.060866:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949019.060870:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060874:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060879:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949019.060885:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949019.060886:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949019.060891:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:1.0:1784949019.060896:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949019.060897:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949019.060898:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949019.060900:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000010:1.0:1784949019.060902:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000100:00000001:0.0:1784949019.060930:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.060934:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949019.060936:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949019.060941:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784949019.060945:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949019.060950:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784949019.060951:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 757 00000400:00000200:1.0:1784949019.060953:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000400:00000010:1.0:1784949019.060958:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000100:00000040:2.0:1784949019.060959:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949019.060963:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770431568 : -115539939120048 : ffff96eac2f8ac50) 00000800:00000001:0.0:1784949019.060966:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949019.060967:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1784949019.060971:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000068fd7440 x1871651289668480/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:680/0 lens 224/0 e 0 to 0 dl 1784949030 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949019.060973:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:1.0:1784949019.060975:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949019.060980:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949019.060997:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949019.061001:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949019.061005:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1784949019.061006:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000100:00000001:2.0:1784949019.061012:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949019.061013:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439c00 00000100:00000040:1.0:1784949019.061013:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949019.061017:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000400:00000200:0.0:1784949019.061025:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 165232 00000100:00000001:1.0:1784949019.061030:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949019.061031:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000068fd7440 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289668480:12345-0@lo:400: 00000800:00000001:0.0:1784949019.061035:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949019.061037:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:2.0:1784949019.061040:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668480 00000020:00000001:2.0:1784949019.061045:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:0.0:1784949019.061046:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949019.061051:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784949019.061052:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949019.061054:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949019.061058:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:0.0:1784949019.061063:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949019.061063:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784949019.061068:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949019.061071:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949019.061073:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:2.0:1784949019.061074:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949019.061078:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:0.0:1784949019.061079:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:2.0:1784949019.061081:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949019.061085:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000010:0.0:1784949019.061087:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000487ebca5. 00000100:00000001:2.0:1784949019.061088:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949019.061092:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:0.0:1784949019.061094:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000487ebca5 x1871651289668608 msgsize 224 02000000:00000010:2.0:1784949019.061098:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d78419ed. 02000000:00000001:2.0:1784949019.061103:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949019.061106:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949019.061110:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:2.0:1784949019.061111:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949019.061118:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668480 00010000:00000001:2.0:1784949019.061121:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949019.061125:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000068fd7440 x1871651289668480/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:680/0 lens 224/224 e 0 to 0 dl 1784949030 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949019.061130:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000001:2.0:1784949019.061133:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949019.061136:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949019.061139:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=41 v=5 (1 1 1 1) 00000800:00000001:0.0:1784949019.061140:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949019.061147:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949019.061147:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949019.061153:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949019.061156:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949019.061158:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:2.0:1784949019.061159:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949019.061163:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949019.061164:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668608 00000100:00000001:2.0:1784949019.061171:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949019.061171:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949019.061176:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:2.0:1784949019.061181:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949019.061181:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949019.061187:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 02000000:00000001:3.0:1784949019.061187:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949019.061195:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289668480, offset 224 00000100:00100000:3.0:1784949019.061195:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668608 00000020:00000001:3.0:1784949019.061199:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949019.061202:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:2.0:1784949019.061208:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:3.0:1784949019.061208:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949019.061213:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949019.061216:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784949019.061220:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:3.0:1784949019.061222:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949019.061227:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784949019.061230:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439b80 00000020:00000010:3.0:1784949019.061231:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000226948e6. 00000100:00000001:3.0:1784949019.061236:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949019.061240:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8b49 [1] + 224 00000020:00000010:3.0:1784949019.061242:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000226948e6. 00000400:00000200:2.0:1784949019.061248:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949019.061249:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000070ada26c. 00000400:00000200:2.0:1784949019.061255:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:3.0:1784949019.061255:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000fe2862da. 00000100:00000001:2.0:1784949019.061258:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949019.061261:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000200:2.0:1784949019.061263:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949019.061271:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:2.0:1784949019.061275:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949019.061275:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949019.061279:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949019.061283:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949019.061292:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949019.061295:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949019.061296:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949019.061301:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949019.061307:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000100:00000001:3.0:1784949019.061309:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949019.061310:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949019.061316:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949019.061319:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949019.061324:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949019.061325:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:2.0:1784949019.061329:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949019.061329:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000001:2.0:1784949019.061335:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949019.061338:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.061341:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949019.061342:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 733 00000020:00000001:2.0:1784949019.061343:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949019.061345:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949019.061348:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784949019.061350:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949019.061351:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:2.0:1784949019.061351:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000068fd7440 x1871651289668480/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:680/0 lens 224/224 e 0 to 0 dl 1784949030 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949019.061355:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949019.061356:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554977920 : -115544154573696 : ffff96e9c7b5f480) 00000100:00000001:1.0:1784949019.061357:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784949019.061360:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949019.061365:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000b3b2e3c. 00000100:00000001:1.0:1784949019.061366:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949019.061370:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000487ebca5 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:680/0 lens 224/0 e 0 to 0 dl 1784949030 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949019.061375:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000100:00100000:2.0:1784949019.061380:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000068fd7440 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289668480:12345-0@lo:400: Request processed in 373us (802us total) trans 0 rc 0/0 00000400:00000010:0.0:1784949019.061382:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000100:00000001:0.0:1784949019.061391:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949019.061393:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:2.0:1784949019.061398:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 757 00000020:00000010:3.0:1784949019.061399:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000226948e6. 00000100:00000200:0.0:1784949019.061401:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949019.061405:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949019.061406:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949019.061411:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000010:3.0:1784949019.061414:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000226948e6. 00000100:00000001:2.0:1784949019.061415:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949019.061422:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ce523b1b. 00000100:00000200:0.0:1784949019.061428:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949019.061429:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000487ebca5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289668608:12345-192.168.202.137@tcp:400: 00000020:00000010:2.0:1784949019.061430:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949019.061437:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 00000100:00000200:3.0:1784949019.061441:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668608 00000020:00000001:3.0:1784949019.061445:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000040:2.0:1784949019.061446:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:3.0:1784949019.061450:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:2.0:1784949019.061453:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:3.0:1784949019.061455:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.061458:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:2.0:1784949019.061458:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d78419ed. 00000020:00000001:3.0:1784949019.061459:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949019.061464:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:0.0:1784949019.061465:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949019.061465:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949019.061470:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949019.061471:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949019.061472:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:2.0:1784949019.061474:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:3.0:1784949019.061475:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949019.061477:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949019.061481:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:2.0:1784949019.061481:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f4e53d4f. 00000020:00000001:3.0:1784949019.061481:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784949019.061487:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949019.061487:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:2.0:1784949019.061489:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c1d3a6d6. 00000020:00000001:3.0:1784949019.061491:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949019.061493:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949019.061496:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00001000:0.0:1784949019.061499:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=41 v=5 (5 5 5 5) 00000100:00000001:3.0:1784949019.061501:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949019.061505:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00001000:0.0:1784949019.061507:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=41 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949019.061511:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:3.0:1784949019.061511:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000ea4643f9. 00000100:00000001:0.0:1784949019.061514:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949019.061516:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949019.061520:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784949019.061520:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949019.061525:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949019.061525:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.061531:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949019.061532:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668608 00010000:00000001:3.0:1784949019.061538:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:0.0:1784949019.061540:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:3.0:1784949019.061544:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000487ebca5 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:680/0 lens 224/224 e 0 to 0 dl 1784949030 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949019.061567:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784949019.061571:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949019.061572:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00010000:00000001:3.0:1784949019.061574:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.061575:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784949019.061581:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=36 v=5 (1 1 1 1) 00000100:00000040:0.0:1784949019.061585:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949019.061588:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949019.061594:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949019.061599:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:0.0:1784949019.061604:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289668480:0@lo:400: 02000000:00000001:3.0:1784949019.061604:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949019.061608:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.061612:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949019.061613:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949019.061619:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949019.061622:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949019.061625:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b16a92e x1871651289668480/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949019.061627:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 00000100:00000200:3.0:1784949019.061634:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289668608, offset 224 00000100:00000001:0.0:1784949019.061637:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949019.061641:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1784949019.061642:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784949019.061644:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949019.061650:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949019.061654:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:3.0:1784949019.061654:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000020:00000040:0.0:1784949019.061658:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949019.061663:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:3.0:1784949019.061664:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3509:3509:243:4294967295] 192.168.202.137@tcp LPNI seq info [2263:2263:8:4294967295] 02000000:00000010:0.0:1784949019.061666:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fd30a9ad. 02000000:00000001:0.0:1784949019.061674:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949019.061678:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949019.061681:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b16a92e. 00000400:00000200:3.0:1784949019.061686:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949019.061688:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949019.061692:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949019.061697:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949019.061698:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949019.061705:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000100:00000001:0.0:1784949019.061706:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949019.061709:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949019.061714:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.0:1784949019.061714:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949019.061718:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949019.061723:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949019.061727:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949019.061741:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949019.061744:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949019.061746:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949019.061747:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949019.061749:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949019.061751:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000487ebca5 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:680/0 lens 224/224 e 0 to 0 dl 1784949030 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949019.061759:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000487ebca5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289668608:12345-192.168.202.137@tcp:400: Request processed in 365us (664us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949019.061766:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 733 00000100:00000040:3.0:1784949019.061769:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949019.061772:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949019.061773:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949019.061775:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000fe2862da. 00000020:00000010:3.0:1784949019.061779:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949019.061782:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000070ada26c. 00000020:00000040:3.0:1784949019.061785:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949019.061787:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949019.061878:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949019.061881:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:1.0:1784949019.061884:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949019.061889:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949019.061892:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000400:00000010:1.0:1784949019.061893:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000100:00000001:1.0:1784949019.061896:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949019.061897:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949019.061902:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949019.061929:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949019.061931:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949019.061935:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949019.061940:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949019.061947:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439c00 00000400:00000200:1.0:1784949019.061952:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8b51 [1] + 224 00000800:00000001:1.0:1784949019.061955:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949019.061960:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949019.061962:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949019.061964:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949019.061967:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949019.061970:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949019.061977:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949019.061990:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949019.061994:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949019.061996:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949019.062023:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949019.062026:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949019.062029:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949019.062031:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949019.062032:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000400:00000200:3.0:1784949019.062038:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:3.0:1784949019.062039:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000100:00000001:3.0:1784949019.062041:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949019.062045:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949019.062052:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949019.062058:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949019.062059:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949019.062061:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949019.062062:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949019.062063:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949019.062065:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949019.062066:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949019.062070:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=36 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949019.062072:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=36 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949019.062074:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949019.062075:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949019.062077:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949019.062079:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949019.062085:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949019.062087:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949019.062089:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949019.062090:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949019.062091:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949019.062093:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949019.062104:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949019.062105:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949019.062107:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949019.062109:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949019.062114:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000067c965f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289668608:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949019.062117:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949019.062118:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949019.062120:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000067c965f9 x1871651289668608/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949019.062125:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949019.062126:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949019.062128:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000015713dee. 02000000:00000001:3.0:1784949019.062130:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949019.062132:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949019.062133:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949019.062134:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949019.062136:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fbc83221. 02000000:00000001:3.0:1784949019.062138:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949019.062139:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949019.062140:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000067c965f9. 00000100:00000001:3.0:1784949019.062144:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949019.062145:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949019.062146:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949019.062149:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949019.062150:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949019.062152:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949019.062153:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949019.102989:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305958442, at 4305958443 00000100:00000040:1.0:1784949019.102989:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305958442, at 4305958443 00000100:00000040:3.0:1784949019.102992:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305958442, at 4305958443 00000100:00000040:2.0:1784949019.102997:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305958442, at 4305958443 00000100:00000010:0.0:1784949019.103006:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000068c34307. 00000100:00000010:1.0:1784949019.103007:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077930f7d. 00000100:00000010:3.0:1784949019.103010:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007024d30a. 00000100:00000010:2.0:1784949019.103013:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00010000:00000001:2.0:1784949019.892009:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949019.892017:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949019.892024:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949019.892025:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949019.892083:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949019.892088:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949019.892092:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949020.084006:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949020.084021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949020.084024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949020.084031:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949020.084034:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949020.084037:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949020.084040:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949020.915995:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949020.916001:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949020.916006:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949020.916007:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949020.916045:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949020.916048:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949020.916051:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949021.108034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949021.108051:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949021.108055:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949021.108063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949021.108067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949021.108072:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949021.108076:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949021.300035:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000800:00000010:1.0:1784949021.300042:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000200:1.0:1784949021.300051:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949021.300059:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949021.300068:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11294 health :900 00000400:00000200:1.0:1784949021.300074:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949021.300078:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:1.0:1784949021.300082:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:3.0:1784949021.300148:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949021.300158:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:3.0:1784949021.300165:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949021.300167:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949021.300171:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000010:3.0:1784949021.300173:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000400:00000200:3.0:1784949021.300178:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949021.300182:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:3.0:1784949021.300186:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000cf77ef26. 00000400:00000200:3.0:1784949021.300192:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:3.0:1784949021.300197:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:3.0:1784949021.300203:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00010000:00000001:2.0:1784949021.940022:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949021.940031:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949021.940040:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949021.940044:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949021.940047:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949021.940050:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949021.940054:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949021.940056:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949021.940059:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949021.940061:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949021.940195:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949021.940204:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949021.940210:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949022.132050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949022.132068:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949022.132072:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949022.132079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949022.132084:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949022.132089:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949022.132093:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949022.324076:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949022.324086:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000090a03b90. 00000400:00000010:1.0:1784949022.324099:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000072ce5ef0. 00000400:00000010:1.0:1784949022.324106:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000010:1.0:1784949022.324116:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d4c102f3. 00000800:00000100:2.0:1784949022.324123:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.10@tcp 00000400:00000200:1.0:1784949022.324131:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949022.324134:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:2.0:1784949022.324144:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949022.324154:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949022.324158:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949022.324170:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949022.324176:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3510:3510:244:4294967295] 192.168.202.137@tcp LPNI seq info [2264:2264:8:4294967295] 00000400:00000200:1.0:1784949022.324184:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949022.324192:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949022.324199:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949022.324205:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000400:00000200:2.0:1784949022.324206:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11295 health :800 00000400:00000200:2.0:1784949022.324229:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000200:1.0:1784949022.324230:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784949022.324244:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000800:00000200:1.0:1784949022.324251:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949022.324260:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949022.324263:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:3.0:1784949022.324354:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949022.324366:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:3.0:1784949022.324375:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949022.324377:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949022.324380:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000010:3.0:1784949022.324384:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000400:00000200:3.0:1784949022.324389:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949022.324394:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:3.0:1784949022.324399:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ee879bbd. 00000400:00000200:3.0:1784949022.324404:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-110 00000400:00000200:3.0:1784949022.324407:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -110 00000400:00000200:3.0:1784949022.324412:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000800:00000200:0.0:1784949022.324506:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949022.324515:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:0.0:1784949022.324525:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949022.324531:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949022.324538:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949022.324545:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949022.324550:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949022.324551:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949022.324557:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949022.324565:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949022.324575:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949022.324592:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949022.324604:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949022.324616:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949022.324622:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3511:3511:245:4294967295] 192.168.202.137@tcp LPNI seq info [2265:2265:8:4294967295] 00000400:00000200:2.0:1784949022.324633:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949022.324639:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949022.324644:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005523afda. 00000800:00000200:2.0:1784949022.324650:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949022.324656:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949022.324660:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005523afda type 3, nob 144 niov 1 nkiov 1 00000800:00000200:2.0:1784949022.324809:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784949022.324813:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784949022.324815:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005523afda. 00000400:00000200:2.0:1784949022.324834:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949022.324836:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949022.324846:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949022.324847:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949022.324860:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949022.324864:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949022.324869:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949022.324879:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784949022.324897:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8b65 00000800:00000001:1.0:1784949022.324933:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949022.324944:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949022.324947:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949022.324952:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949022.324958:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:1.0:1784949022.324961:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000400:00000200:1.0:1784949022.324965:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949022.324969:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000090a03b90. 00000400:00000010:1.0:1784949022.324972:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000072ce5ef0. 00000800:00000001:1.0:1784949022.324979:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949022.324982:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949022.964039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949022.964049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949022.964056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949022.964058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949022.964166:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949022.964173:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949022.964177:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949023.156033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949023.156049:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949023.156052:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949023.156059:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949023.156063:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949023.156067:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949023.156070:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949023.348046:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000800:00000010:2.0:1784949023.348054:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000076202100. 00000400:00000200:2.0:1784949023.348068:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949023.348078:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949023.348090:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949023.348107:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:2.0:1784949023.348112:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:1.0:1784949023.348178:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:3.0:1784949023.348187:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949023.348200:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000010:1.0:1784949023.348203:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000090a03b90. 00000400:00000001:3.0:1784949023.348210:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949023.348214:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949023.348224:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:3.0:1784949023.348228:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000400:00000010:1.0:1784949023.348233:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000072ce5ef0. 00000400:00000200:3.0:1784949023.348236:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949023.348248:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:1.0:1784949023.348255:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000010:3.0:1784949023.348255:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000307ddfde. 00000400:00000200:3.0:1784949023.348293:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:3.0:1784949023.348298:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000400:00000200:3.0:1784949023.348303:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00000400:00000010:1.0:1784949023.348317:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000934e2586. 00000400:00000200:1.0:1784949023.348328:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949023.348352:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949023.348361:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949023.348366:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3512:3512:246:4294967295] 192.168.202.137@tcp LPNI seq info [2266:2266:8:4294967295] 00000400:00000200:1.0:1784949023.348375:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949023.348384:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949023.348390:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949023.348397:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000800:00000200:1.0:1784949023.348406:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949023.348418:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949023.348427:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949023.348459:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11297 health :900 00000800:00000200:0.0:1784949023.348685:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949023.348693:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:0.0:1784949023.348702:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949023.348711:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949023.348718:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949023.348723:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784949023.348739:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949023.348753:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949023.348790:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949023.348800:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949023.348808:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949023.348825:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949023.348837:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949023.348855:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949023.348865:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3513:3513:246:4294967295] 192.168.202.137@tcp LPNI seq info [2267:2267:8:4294967295] 00000400:00000200:0.0:1784949023.348885:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949023.348895:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949023.348957:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d1b12746. 00000800:00000200:0.0:1784949023.348970:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949023.348982:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949023.348987:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d1b12746 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949023.349053:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949023.349128:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949023.349138:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d1b12746. 00000800:00000001:2.0:1784949023.349142:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949023.349148:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949023.349158:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949023.349164:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949023.349165:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949023.349167:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949023.349185:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949023.349204:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949023.349227:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8b69 00000800:00000001:2.0:1784949023.349237:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949023.349254:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949023.349259:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949023.349265:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949023.349274:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:2.0:1784949023.349278:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000400:00000200:2.0:1784949023.349284:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949023.349289:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000090a03b90. 00000400:00000010:2.0:1784949023.349294:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000072ce5ef0. 00000800:00000001:2.0:1784949023.349305:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949023.349310:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949023.988047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949023.988054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949023.988060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949023.988062:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949023.988125:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949023.988129:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949023.988132:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949024.180012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949024.180026:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949024.180028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.180035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949024.180039:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949024.180042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.180044:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949024.180051:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949024.180075:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949024.180082:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006e540bde. 00000020:00000040:3.0:1784949024.180091:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949024.180099:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949024.180103:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949024.180113:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949024.180141:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:3.0:1784949024.180154:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949024.180174:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949024.180199:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949024.180204:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949024.180215:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e540bde] to pc [ptlrpcd_00_01+1] req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949024.180265:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949024.180288:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949024.180292:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949024.180295:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949024.180299:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949024.180303:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001f1afd3c. 00000020:00000040:3.0:1784949024.180308:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949024.180312:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949024.180314:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949024.180317:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949024.180324:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:2.0:1784949024.180328:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949024.180330:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.180336:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949024.180343:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949024.180346:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784949024.180353:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949024.180374:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949024.180378:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.180380:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949024.180384:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949024.180389:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1afd3c] to pc [ptlrpcd_00_02+2] req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949024.180390:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949024.180394:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949024.180401:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949024.180409:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289668736:0@lo:400: 00000100:00000001:2.0:1784949024.180414:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949024.180421:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949024.180423:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949024.180427:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949024.180436:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 02000000:00000001:2.0:1784949024.180439:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949024.180449:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000062d113c0. 00000400:00000010:2.0:1784949024.180460:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000100:00000200:2.0:1784949024.180485:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668736, portal 25 00000100:00000001:2.0:1784949024.180487:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949024.180489:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554957056 : -115544154594560 : ffff96e9c7b5a300) 00000100:00000040:2.0:1784949024.180494:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949024.180501:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949024.180511:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949024.180515:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000100:00000200:2.0:1784949024.180520:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289668736, offset 0 00000100:00000001:3.0:1784949024.180520:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949024.180527:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11301) 00000100:00000001:1.0:1784949024.180530:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949024.180531:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949024.180539:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949024.180544:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949024.180549:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:1.0:1784949024.180555:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949024.180566:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439c80 00000100:00000001:1.0:1784949024.180582:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1784949024.180582:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x81c9 [2] + 0 00000100:00000001:1.0:1784949024.180590:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949024.180591:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002bff2598. 02000000:00000001:1.0:1784949024.180605:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784949024.180605:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d4d2d632 02000000:00000001:1.0:1784949024.180609:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949024.180614:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949024.180615:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949024.180632:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00100000:1.0:1784949024.180636:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000001f1afd3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289668864:192.168.202.137@tcp:400: 00000400:00000200:2.0:1784949024.180639:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4d2d632 00000100:00000001:3.0:1784949024.180641:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784949024.180646:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d4d2d632. 00000100:00000001:3.0:1784949024.180647:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949024.180657:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949024.180658:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949024.180664:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784949024.180665:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949024.180667:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1784949024.180668:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000065a766c3 x1871651289668736 msgsize 224 00000100:00000001:0.0:1784949024.180670:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949024.180674:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949024.180680:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1784949024.180686:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 02000000:00000010:1.0:1784949024.180690:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000bf393d23. 00000100:00000040:2.0:1784949024.180691:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 02000000:00000001:1.0:1784949024.180702:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949024.180708:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000011e3ad9. 00000100:00000001:2.0:1784949024.180709:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949024.180720:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:1.0:1784949024.180721:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000010:2.0:1784949024.180723:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000100:00000001:2.0:1784949024.180729:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949024.180734:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949024.180736:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668864, portal 10 00000100:00000001:1.0:1784949024.180745:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949024.180748:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554948992 : -115544154602624 : ffff96e9c7b58380) 00000100:00000001:2.0:1784949024.180772:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949024.180777:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949024.180782:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949024.180799:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949024.180803:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949024.180809:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668736 00000100:00000040:1.0:1784949024.180810:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949024.180824:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.180828:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949024.180836:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949024.180837:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949024.180842:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949024.180847:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.180848:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305963562 02000000:00000001:0.0:1784949024.180851:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949024.180854:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000ef8f137f. 00000100:00100000:0.0:1784949024.180858:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668736 00000020:00000001:0.0:1784949024.180864:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949024.180864:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000040:0.0:1784949024.180869:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949024.180876:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949024.180878:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305963562 00000020:00000040:0.0:1784949024.180881:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:2.0:1784949024.180884:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e19b789f. 00000020:00000001:0.0:1784949024.180887:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949024.180892:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1784949024.180894:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949024.180901:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305963562 00000100:00000001:0.0:1784949024.180903:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:2.0:1784949024.180933:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a2179890. 00000100:00000001:2.0:1784949024.180937:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949024.180946:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305963563 00000100:00000010:2.0:1784949024.180951:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001cae17e9. 00000100:00000001:2.0:1784949024.180956:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000010:0.0:1784949024.180960:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:2.0:1784949024.180964:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949024.180970:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949024.180971:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949024.180978:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:2.0:1784949024.180978:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.180985:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949024.180989:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:2.0:1784949024.180990:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949024.180995:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:0.0:1784949024.181013:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003fada1b7. 00000400:00000010:1.0:1784949024.181015:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000100:00000001:2.0:1784949024.181016:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949024.181022:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:1.0:1784949024.181034:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289668864, offset 0 00000020:00000010:0.0:1784949024.181036:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949024.181036:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.181053:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949024.181058:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949024.181062:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949024.181086:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:1.0:1784949024.181088:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949024.181090:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949024.181096:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949024.181099:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949024.181113:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949024.181115:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949024.181121:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949024.181127:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949024.181131:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3514:3514:246:4294967295] 192.168.202.137@tcp LPNI seq info [2268:2268:8:4294967295] 00000100:00000001:0.0:1784949024.181138:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949024.181154:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949024.181163:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949024.181164:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949024.181173:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000010:1.0:1784949024.181177:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000100:00100000:0.0:1784949024.181186:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 758 00000800:00000200:1.0:1784949024.181193:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:0.0:1784949024.181194:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949024.181203:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770425424 : -115539939126192 : ffff96eac2f89450) 00000100:00000040:0.0:1784949024.181219:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000065a766c3 x1871651289668736/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:685/0 lens 224/0 e 0 to 0 dl 1784949035 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949024.181221:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949024.181234:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949024.181255:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949024.181261:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:1.0:1784949024.181264:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949024.181267:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949024.181271:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2135 usecs (3 jiffies) 00000100:00000001:1.0:1784949024.181287:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305963524 00000020:00000010:0.0:1784949024.181292:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:1.0:1784949024.181293:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305963562 00000100:00000001:1.0:1784949024.181299:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305963524 00000100:00100000:0.0:1784949024.181308:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000065a766c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289668736:12345-0@lo:400: 00000100:00000001:1.0:1784949024.181309:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305963562 00000100:00000001:1.0:1784949024.181321:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305963524 00000100:00000200:0.0:1784949024.181323:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668736 00000100:00000001:1.0:1784949024.181327:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305963562 00000020:00000001:0.0:1784949024.181330:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949024.181332:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305963524 00000020:00000001:0.0:1784949024.181335:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949024.181337:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305963563 00000020:00000001:0.0:1784949024.181339:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949024.181341:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949024.181348:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949024.181350:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949024.181352:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784949024.181355:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949024.181358:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949024.181360:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949024.181362:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949024.181370:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784949024.181372:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949024.181386:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:0.0:1784949024.181393:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949024.181394:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949024.181399:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784949024.181400:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949024.181406:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949024.181408:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949024.181416:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949024.181417:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949024.181431:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949024.181438:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006296bcc7. 02000000:00000001:0.0:1784949024.181444:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949024.181448:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949024.181451:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949024.181456:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668736 00010000:00000001:0.0:1784949024.181461:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949024.181484:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000065a766c3 x1871651289668736/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:685/0 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949024.181490:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949024.181494:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949024.181499:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=46 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949024.181504:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949024.181506:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949024.181508:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949024.181510:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949024.181512:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949024.181515:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949024.181517:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949024.181520:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00000200:0.0:1784949024.181524:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289668736, offset 224 00000400:00000200:0.0:1784949024.181530:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784949024.181540:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784949024.181548:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439c80 00000400:00000200:0.0:1784949024.181553:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8b6d [1] + 224 00000400:00000200:0.0:1784949024.181558:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949024.181561:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949024.181570:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949024.181577:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949024.181578:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949024.181583:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:1.0:1784949024.181594:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000100:00000040:0.0:1784949024.181599:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949024.181600:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949024.181604:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949024.181608:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949024.181617:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949024.181623:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949024.181626:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784949024.181633:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000200:3.0:1784949024.181634:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949024.181641:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784949024.181642:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000100:00000001:2.0:1784949024.181644:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949024.181652:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439d00 00000100:00000001:1.0:1784949024.181655:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:2.0:1784949024.181656:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949024.181661:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949024.181663:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000062d113c0. 00000100:00000200:1.0:1784949024.181667:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949024.181669:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784949024.181675:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949024.181680:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949024.181686:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949024.181689:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 165456 00000400:00000200:0.0:1784949024.181709:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000800:00000001:3.0:1784949024.181710:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949024.181720:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949024.181721:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000400:00000200:3.0:1784949024.181731:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949024.181732:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949024.181732:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:2.0:1784949024.181737:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949024.181741:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000040:1.0:1784949024.181745:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949024.181750:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949024.181766:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:0.0:1784949024.181782:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949024.181792:0:337113:0:(events.c:286:request_in_callback()) Process entered 00010000:00000001:0.0:1784949024.181793:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949024.181801:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949024.181801:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784949024.181802:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949024.181807:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.181807:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949024.181809:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000010:3.0:1784949024.181811:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b44b3d82. 00000100:00000040:0.0:1784949024.181819:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000065a766c3 x1871651289668736/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:685/0 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949024.181824:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949024.181825:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:3.0:1784949024.181825:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000b44b3d82 x1871651289668864 msgsize 224 02000000:00000001:2.0:1784949024.181835:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949024.181839:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:0.0:1784949024.181841:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000065a766c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289668736:12345-0@lo:400: Request processed in 583us (1169us total) trans 0 rc 0/0 00000100:00000001:2.0:1784949024.181842:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949024.181848:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:2.0:1784949024.181849:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949024.181856:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=46 v=5 (5 5 5 5) 00000100:00100000:0.0:1784949024.181861:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 758 00000100:00001000:2.0:1784949024.181863:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=46 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949024.181863:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784949024.181865:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949024.181871:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949024.181872:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949024.181874:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949024.181875:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:3.0:1784949024.181875:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784949024.181877:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949024.181881:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949024.181882:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003fada1b7. 00000800:00000001:3.0:1784949024.181883:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.181885:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949024.181890:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949024.181891:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:2.0:1784949024.181895:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949024.181898:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668864 00000020:00000010:0.0:1784949024.181900:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 02000000:00000001:1.0:1784949024.181946:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000040:0.0:1784949024.181948:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949024.181952:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949024.181956:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949024.181957:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949024.181958:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784949024.181965:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006296bcc7. 00000100:00000001:2.0:1784949024.181968:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1784949024.181969:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949024.181970:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949024.181971:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949024.181977:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949024.181980:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668864 00000100:00000200:0.0:1784949024.181982:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:2.0:1784949024.181984:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949024.181990:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:0.0:1784949024.181992:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000b3b2e3c. 00000020:00000040:1.0:1784949024.181997:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:0.0:1784949024.182001:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a21d030b. 00000020:00000001:1.0:1784949024.182006:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949024.182013:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00100000:2.0:1784949024.182017:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289668736:0@lo:400: 00000020:00000001:1.0:1784949024.182018:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949024.182028:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949024.182030:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182033:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949024.182034:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784949024.182037:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006e540bde x1871651289668736/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949024.182038:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784949024.182047:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949024.182058:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00000001:2.0:1784949024.182070:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000010:1.0:1784949024.182073:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 02000000:00000001:2.0:1784949024.182073:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949024.182081:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 00000020:00000010:1.0:1784949024.182083:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002afdbc54. 02000000:00000001:2.0:1784949024.182092:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:1.0:1784949024.182093:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000020:00000001:2.0:1784949024.182097:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949024.182100:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949024.182103:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:1.0:1784949024.182106:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000010:2.0:1784949024.182107:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 00000100:00000001:1.0:1784949024.182116:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:2.0:1784949024.182118:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1784949024.182120:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:2.0:1784949024.182120:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949024.182125:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006e540bde. 00000100:00000001:1.0:1784949024.182126:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182139:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949024.182142:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949024.182142:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182151:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949024.182152:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949024.182158:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.182161:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949024.182163:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.182165:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949024.182166:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949024.182169:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949024.182180:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949024.182182:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949024.182187:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 734 00000100:00000040:1.0:1784949024.182190:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949024.182193:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554957952 : -115544154593664 : ffff96e9c7b5a680) 00000100:00000040:1.0:1784949024.182198:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b44b3d82 x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:685/0 lens 224/0 e 0 to 0 dl 1784949035 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949024.182205:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949024.182207:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784949024.182209:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949024.182210:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00100000:1.0:1784949024.182213:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b44b3d82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289668864:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949024.182216:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668864 00000020:00000001:1.0:1784949024.182218:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949024.182221:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949024.182223:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949024.182225:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949024.182226:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949024.182229:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949024.182231:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949024.182234:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949024.182235:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949024.182237:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949024.182239:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949024.182243:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949024.182245:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949024.182248:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000094500b89. 02000000:00000001:1.0:1784949024.182249:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949024.182251:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949024.182253:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949024.182255:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668864 00010000:00000001:1.0:1784949024.182257:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949024.182260:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b44b3d82 x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:685/0 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949024.182265:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949024.182267:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949024.182269:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=41 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949024.182272:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949024.182274:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949024.182276:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949024.182278:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949024.182279:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949024.182281:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949024.182283:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949024.182285:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000100:00000200:1.0:1784949024.182287:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289668864, offset 224 00000400:00000200:1.0:1784949024.182290:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949024.182297:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949024.182302:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3515:3515:246:4294967295] 192.168.202.137@tcp LPNI seq info [2269:2269:8:4294967295] 00000400:00000200:1.0:1784949024.182308:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949024.182311:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949024.182313:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000800:00000200:1.0:1784949024.182316:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949024.182320:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949024.182323:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949024.182333:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949024.182335:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949024.182336:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949024.182338:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949024.182339:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949024.182341:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b44b3d82 x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:685/0 lens 224/224 e 0 to 0 dl 1784949035 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949024.182348:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b44b3d82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289668864:12345-192.168.202.137@tcp:400: Request processed in 141us (521us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949024.182353:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 734 00000100:00000040:1.0:1784949024.182357:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949024.182360:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949024.182362:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949024.182365:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002afdbc54. 00000020:00000010:1.0:1784949024.182368:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949024.182371:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00000020:00000040:1.0:1784949024.182374:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949024.182376:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949024.182437:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949024.182442:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000400:00000200:2.0:1784949024.182445:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949024.182451:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949024.182452:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949024.182458:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:2.0:1784949024.182460:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000800:00000001:1.0:1784949024.182464:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182464:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949024.182466:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949024.182475:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949024.182481:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949024.182490:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949024.182499:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439d00 00000400:00000200:1.0:1784949024.182504:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8b75 [1] + 224 00000800:00000001:1.0:1784949024.182508:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949024.182515:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949024.182517:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949024.182519:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949024.182524:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949024.182526:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949024.182534:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949024.182547:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949024.182552:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949024.182553:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182576:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949024.182578:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949024.182582:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949024.182583:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949024.182585:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000011e3ad9. 00000400:00000200:2.0:1784949024.182588:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:2.0:1784949024.182589:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:2.0:1784949024.182591:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949024.182595:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949024.182601:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949024.182606:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949024.182607:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182609:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949024.182610:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949024.182611:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949024.182612:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949024.182614:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949024.182618:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=41 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949024.182620:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=41 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949024.182622:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949024.182623:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949024.182624:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949024.182626:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949024.182632:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.182634:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949024.182635:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949024.182637:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949024.182638:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949024.182640:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949024.182644:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949024.182645:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949024.182646:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949024.182648:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949024.182652:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000001f1afd3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289668864:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949024.182655:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182656:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949024.182658:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000001f1afd3c x1871651289668864/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949024.182662:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949024.182663:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949024.182665:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000bf393d23. 02000000:00000001:2.0:1784949024.182667:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949024.182668:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949024.182669:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949024.182670:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949024.182671:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:2.0:1784949024.182673:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949024.182674:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949024.182675:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000001f1afd3c. 00000100:00000001:2.0:1784949024.182685:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949024.182686:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182687:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949024.182690:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949024.182691:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949024.182692:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949024.182694:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949024.223004:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305963562, at 4305963563 00000100:00000010:0.0:1784949024.223019:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000ef8f137f. 00000100:00000040:1.0:1784949024.223025:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305963562, at 4305963563 00000100:00000010:1.0:1784949024.223048:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e19b789f. 00000100:00000040:2.0:1784949024.223978:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305963562, at 4305963564 00000100:00000010:2.0:1784949024.223986:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a2179890. 00000100:00000040:3.0:1784949024.223986:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305963563, at 4305963564 00000100:00000010:3.0:1784949024.224015:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00010000:00000001:2.0:1784949025.012044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949025.012053:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949025.012061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949025.012063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949025.012214:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949025.012219:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949025.012222:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949025.204050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949025.204067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949025.204069:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949025.204090:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949025.204093:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949025.204097:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949025.204100:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949026.035991:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949026.035997:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949026.036003:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949026.036004:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949026.036082:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949026.036088:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949026.036093:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949026.227992:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949026.228004:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949026.228006:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949026.228010:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949026.228013:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949026.228017:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949026.228019:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949026.420019:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.4@tcp 00000800:00000010:2.0:1784949026.420024:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a21d8ef3. 00000400:00000200:2.0:1784949026.420032:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949026.420037:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949026.420043:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11299 health :900 00000400:00000200:2.0:1784949026.420047:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949026.420050:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:2.0:1784949026.420054:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000800:00000100:2.0:1784949026.420074:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.10@tcp 00000800:00000010:2.0:1784949026.420076:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007a3f8609. 00000400:00000200:2.0:1784949026.420078:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949026.420082:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949026.420086:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949026.420088:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:2.0:1784949026.420090:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:3.0:1784949026.420099:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949026.420107:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:3.0:1784949026.420113:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949026.420115:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949026.420117:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000010:3.0:1784949026.420119:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000400:00000200:3.0:1784949026.420123:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949026.420126:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:3.0:1784949026.420129:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000075318cf7. 00000400:00000200:3.0:1784949026.420133:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-110 00000400:00000200:3.0:1784949026.420135:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -110 00000400:00000200:3.0:1784949026.420138:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:3.0:1784949026.420141:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000001:3.0:1784949026.420143:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949026.420144:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949026.420145:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:3.0:1784949026.420146:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:3.0:1784949026.420148:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949026.420149:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:3.0:1784949026.420151:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000b241612. 00000400:00000200:3.0:1784949026.420153:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-110 00000400:00000200:3.0:1784949026.420155:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -110 00000400:00000200:3.0:1784949026.420156:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00010000:00000001:2.0:1784949027.060086:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949027.060093:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949027.060099:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949027.060100:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949027.060146:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949027.060150:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949027.060152:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949027.251991:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949027.252002:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949027.252003:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949027.252006:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949027.252009:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949027.252012:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949027.252013:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949027.443977:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949027.443985:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000048fd39a3. 00000400:00000010:1.0:1784949027.444011:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000378c7327. 00000400:00000010:1.0:1784949027.444019:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000400:00000010:1.0:1784949027.444030:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b03d8085. 00000400:00000200:1.0:1784949027.444036:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949027.444054:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949027.444062:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949027.444067:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3516:3516:248:4294967295] 192.168.202.137@tcp LPNI seq info [2270:2270:8:4294967295] 00000400:00000200:1.0:1784949027.444076:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949027.444098:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949027.444102:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949027.444106:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000800:00000200:1.0:1784949027.444112:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949027.444118:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949027.444122:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949027.444139:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11301 health :800 00000800:00000200:3.0:1784949027.444252:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949027.444258:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000400:00000200:3.0:1784949027.444267:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949027.444271:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949027.444274:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949027.444279:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:2.0:1784949027.444282:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949027.444282:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949027.444286:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949027.444290:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949027.444297:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949027.444310:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949027.444319:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949027.444329:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949027.444334:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3517:3517:248:4294967295] 192.168.202.137@tcp LPNI seq info [2271:2271:8:4294967295] 00000400:00000200:2.0:1784949027.444343:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949027.444347:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949027.444351:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005523afda. 00000800:00000200:2.0:1784949027.444354:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949027.444358:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949027.444361:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005523afda type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949027.444402:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949027.444444:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949027.444449:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005523afda. 00000400:00000200:2.0:1784949027.444454:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949027.444457:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949027.444459:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949027.444463:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949027.444465:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949027.444469:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949027.444470:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949027.444479:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949027.444491:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8b89 00000800:00000001:3.0:1784949027.444496:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949027.444524:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949027.444528:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949027.444535:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949027.444541:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:3.0:1784949027.444544:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000400:00000200:3.0:1784949027.444548:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949027.444551:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000048fd39a3. 00000400:00000010:3.0:1784949027.444556:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000378c7327. 00000800:00000001:3.0:1784949027.444563:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949027.444567:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949028.084014:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949028.084024:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949028.084034:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949028.084037:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949028.084197:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949028.084204:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949028.084209:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949028.276010:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949028.276020:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949028.276037:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949003/real 1784949003] req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949028 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784949028.276075:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949028.276078:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949028.276082:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000085e44521. 00000400:00000200:2.0:1784949028.276091:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:2.0:1784949028.276094:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000100:00000001:2.0:1784949028.276100:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949028.276106:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949028 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949028.276118:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949028 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949028.276129:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949028.276132:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.276136:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949028.276139:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949028.276143:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949028 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949028.276152:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.276162:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949028.276165:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.276169:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949028.276173:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949028.276176:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949028.276180:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949028 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949028.276189:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949028.276194:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.276197:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949028.276199:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.276202:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949028.276206:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949028.276210:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949028.276243:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:2.0:1784949028.276247:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949028.276252:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949028 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949028.276263:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289667712:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784949028.276271:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.276275:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949028.276279:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289667712/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949028 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949028.276291:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949028.276295:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1784949028.276296:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:2.0:1784949028.276300:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d58187af. 00000100:00000001:3.0:1784949028.276311:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:2.0:1784949028.276313:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949028.276319:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000010:3.0:1784949028.276320:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046cb422a. 00000020:00000040:2.0:1784949028.276322:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000040:3.0:1784949028.276330:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949028.276331:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949028.276337:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000b91d4ef5. 00000100:00000001:3.0:1784949028.276340:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949028.276344:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949028.276346:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949028.276348:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949028.276353:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 02000000:00000001:3.0:1784949028.276354:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.276364:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000010:3.0:1784949028.276365:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:2.0:1784949028.276368:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.276373:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.276376:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.276378:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949028.276383:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.276386:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949028.276387:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949028.276390:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949028.276407:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949028.276411:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949028.276419:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046cb422a] to pc [ptlrpcd_00_03+3] req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949028.276463:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949028.276466:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949028.276469:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784949028.276471:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784949028.276473:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784949028.276476:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784949028.276482:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11275 00000100:00000001:0.0:1784949028.276483:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949028.276489:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949028.276491:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1784949028.276494:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000040:0.0:1784949028.276497:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949028.276497:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784949028.276501:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11274 00000100:00000001:2.0:1784949028.276503:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.276513:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949028.276515:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949028.276519:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784949028.276519:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 02000000:00000001:0.0:1784949028.276524:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949028.276526:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 02000000:00000001:0.0:1784949028.276529:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949028.276533:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00100000:0.0:1784949028.276535:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289668992:0@lo:400: 00000100:00000001:3.0:1784949028.276538:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949028.276541:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1784949028.276544:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000400:00000200:1.0:1784949028.276546:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:3.0:1784949028.276547:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:0.0:1784949028.276548:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949028.276552:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949028.276554:0:285579:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1784949028.276555:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949028.276558:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000020:00000040:3.0:1784949028.276559:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000010:0.0:1784949028.276562:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 00000020:00000001:3.0:1784949028.276563:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:0.0:1784949028.276568:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949028.276568:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009f047fe3. 00000020:00000040:3.0:1784949028.276569:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:3.0:1784949028.276575:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:3.0:1784949028.276577:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:1.0:1784949028.276578:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b039801a. 00000400:00000010:0.0:1784949028.276579:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001ee400ea. 00000100:00000040:3.0:1784949028.276582:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949028.276586:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949028.276588:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000010:1.0:1784949028.276591:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000054872d18. 00000100:00000001:3.0:1784949028.276591:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:0.0:1784949028.276592:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000020:00000040:3.0:1784949028.276599:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000200:1.0:1784949028.276601:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00080000:3.0:1784949028.276605:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000200:0.0:1784949028.276608:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289668992, portal 25 00000100:00080000:3.0:1784949028.276608:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:0.0:1784949028.276614:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949028.276615:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:3.0:1784949028.276615:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.276617:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554953472 : -115544154598144 : ffff96e9c7b59500) 02000000:00000001:3.0:1784949028.276621:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1784949028.276626:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1784949028.276627:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949028.276631:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949028.276633:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:3.0:1784949028.276639:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949028.276642:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949028.276644:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949028.276645:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1784949028.276648:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:3.0:1784949028.276650:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949028.276654:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000010:3.0:1784949028.276657:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000400:00000200:1.0:1784949028.276659:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3518:3518:248:4294967295] 1.2.3.4@tcp LPNI seq info [108:108:8:4294967295] 00000400:00000010:0.0:1784949028.276660:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000020:00000040:3.0:1784949028.276663:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:0.0:1784949028.276667:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289668992, offset 0 02000000:00000001:3.0:1784949028.276670:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1784949028.276674:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 02000000:00000001:3.0:1784949028.276674:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949028.276679:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949028.276681:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949028.276686:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1784949028.276690:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 02000000:00000001:3.0:1784949028.276690:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949028.276698:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008142e1f1. 00000800:00000200:1.0:1784949028.276702:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:3.0:1784949028.276705:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949028.276708:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:1.0:1784949028.276709:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c2b2d4f6. 00000100:00100000:3.0:1784949028.276715:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784949028.276735:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439d80 00000800:00000200:1.0:1784949028.276738:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:3.0:1784949028.276748:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1784949028.276751:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x81e9 [2] + 0 00000100:00000001:3.0:1784949028.276753:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1784949028.276753:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:0.0:1784949028.276758:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000382215bc. 00000100:00000040:3.0:1784949028.276762:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006d868603] to pc [ptlrpcd_rcv+-1] req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949028.276765:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000048b052c7 00000400:00000200:1.0:1784949028.276774:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000400:00000200:0.0:1784949028.276775:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949028.276777:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949028.276780:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 00000100:00000040:3.0:1784949028.276782:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1784949028.276784:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949028.276791:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949028.276795:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000040:3.0:1784949028.276798:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:0.0:1784949028.276799:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000048b052c7 00000100:00000001:3.0:1784949028.276801:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1784949028.276803:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000048b052c7. 02000000:00000001:3.0:1784949028.276804:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949028.276808:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949028.276812:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:0.0:1784949028.276813:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949028.276816:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784949028.276816:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949028.276823:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000009c11a764 x1871651289668992 msgsize 224 00000100:00000040:3.0:1784949028.276826:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1784949028.276837:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784949028.276839:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1784949028.276841:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949028.276843:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949028.276849:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_00+0] req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949028.276855:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949028.276863:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:0.0:1784949028.276865:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000001:3.0:1784949028.276868:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.276870:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949028.276877:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949028.276877:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11305) 00000100:00000001:0.0:1784949028.276887:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949028.276891:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949028.276897:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.276899:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949028.276900:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949028.276926:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949028.276929:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949028.276931:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949028.276935:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949028.276936:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305967659 00000100:00000010:0.0:1784949028.276944:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:3.0:1784949028.276946:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949028.276948:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.276950:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949028.276951:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949028.276952:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949028.276954:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949028.276955:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949028.276976:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1784949028.276978:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.276980:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305967659 00000100:00000040:2.0:1784949028.276989:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949028.276989:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289669248:192.168.202.137@tcp:400: 00000100:00000010:0.0:1784949028.276991:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:0.0:1784949028.277002:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949028.277006:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949028.277013:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949028.277014:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305967659 00000100:00000001:2.0:1784949028.277015:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1784949028.277018:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.277020:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949028.277024:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 02000000:00000001:3.0:1784949028.277026:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1784949028.277027:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:3.0:1784949028.277034:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000015713dee. 00000100:00000001:0.0:1784949028.277035:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:2.0:1784949028.277035:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949028.277045:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949028.277046:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.277047:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305967659 00000100:00000010:0.0:1784949028.277060:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00100000:2.0:1784949028.277060:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289669120:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949028.277068:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949028.277070:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949028.277077:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.277078:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949028.277080:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.277083:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.277086:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.277088:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.277091:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949028.277094:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.277099:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949028.277102:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949028.277103:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949028.277107:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949028.277110:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949028.277116:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949028.277116:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949028.277123:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949028.277123:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.277128:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.277129:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.277139:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.277169:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949028.277175:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949028.277187:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949028.277193:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289668992 02000000:00000001:1.0:1784949028.277198:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949028.277200:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949028.277204:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949028.277205:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000df521cee. 02000000:00000001:1.0:1784949028.277215:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949028.277219:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000100:00100000:1.0:1784949028.277222:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289668992 00000100:00000200:3.0:1784949028.277226:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289669248, portal 10 00000020:00000001:1.0:1784949028.277227:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949028.277231:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949028.277233:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1784949028.277236:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949028.277238:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000020:00000040:1.0:1784949028.277242:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949028.277247:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784949028.277253:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:3.0:1784949028.277253:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949028.277262:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949028.277269:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:1.0:1784949028.277271:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000040:3.0:1784949028.277278:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949028.277281:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784949028.277284:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000020:00000010:1.0:1784949028.277286:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00000200:3.0:1784949028.277291:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289669248, offset 0 00000020:00000010:1.0:1784949028.277298:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f7a7941e. 00000400:00000200:3.0:1784949028.277307:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000010:1.0:1784949028.277310:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003cf4ade9. 00000020:00000010:1.0:1784949028.277318:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000400:00000200:3.0:1784949028.277319:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949028.277329:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1784949028.277333:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:3.0:1784949028.277334:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949028.277341:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:3.0:1784949028.277345:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784949028.277346:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949028.277352:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949028.277355:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949028.277362:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3519:3519:247:4294967295] 192.168.202.137@tcp LPNI seq info [2272:2272:8:4294967295] 00000100:00000001:1.0:1784949028.277365:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949028.277375:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949028.277382:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949028.277383:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949028.277388:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949028.277396:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949028.277400:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949028.277404:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:3.0:1784949028.277409:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:1.0:1784949028.277415:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 759 00000800:00000200:3.0:1784949028.277418:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000040:1.0:1784949028.277421:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949028.277426:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770428496 : -115539939123120 : ffff96eac2f8a050) 00000100:00000001:3.0:1784949028.277432:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949028.277449:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009c11a764 x1871651289668992/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:689/0 lens 224/0 e 0 to 0 dl 1784949039 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949028.277455:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2136 usecs (3 jiffies) 00000100:00000001:3.0:1784949028.277461:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305967620 00000100:00000001:1.0:1784949028.277465:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949028.277466:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305967659 00000020:00000010:1.0:1784949028.277470:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:3.0:1784949028.277470:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305967620 00000100:00000001:3.0:1784949028.277475:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305967659 00000100:00000001:1.0:1784949028.277476:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949028.277480:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305967620 00000020:00000010:1.0:1784949028.277481:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00000001:3.0:1784949028.277483:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305967659 00000100:00100000:1.0:1784949028.277487:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009c11a764 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289668992:12345-0@lo:400: 00000100:00000001:3.0:1784949028.277487:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305967620 00000100:00000001:3.0:1784949028.277493:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305967659 00000100:00000200:1.0:1784949028.277494:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289668992 00000100:00000001:3.0:1784949028.277496:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949028.277498:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949028.277499:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949028.277503:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949028.277503:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949028.277506:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949028.277507:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949028.277513:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949028.277517:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949028.277517:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949028.277521:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:1.0:1784949028.277522:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949028.277525:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949028.277526:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.277529:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1784949028.277530:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949028.277532:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949028.277533:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949028.277536:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949028.277537:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949028.277537:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:1.0:1784949028.277543:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000010:2.0:1784949028.277544:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000001:1.0:1784949028.277547:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:2.0:1784949028.277550:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784949028.277551:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949028.277556:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e5630c1a. 02000000:00000001:1.0:1784949028.277560:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949028.277561:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949028.277564:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949028.277567:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949028.277567:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000800:00000001:0.0:1784949028.277568:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000040:1.0:1784949028.277573:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289668992 00000400:00000010:2.0:1784949028.277574:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00010000:00000001:1.0:1784949028.277580:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949028.277580:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:0.0:1784949028.277581:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:1.0:1784949028.277585:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009c11a764 x1871651289668992/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:689/0 lens 224/224 e 0 to 0 dl 1784949039 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949028.277586:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:2.0:1784949028.277587:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949028.277591:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949028.277606:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:1.0:1784949028.277617:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949028.277619:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1784949028.277621:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949028.277622:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6439e80 00000100:00001000:1.0:1784949028.277626:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=50 v=5 (1 1 1 1) 00000100:00000040:2.0:1784949028.277626:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949028.277635:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 165680 00000100:00000001:1.0:1784949028.277637:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949028.277642:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:0.0:1784949028.277648:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949028.277648:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:2.0:1784949028.277651:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949028.277654:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784949028.277655:0:337112:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1784949028.277657:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949028.277658:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949028.277661:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949028.277666:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784949028.277666:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949028.277670:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000100:00000200:1.0:1784949028.277676:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289668992, offset 224 00000400:00000200:0.0:1784949028.277677:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949028.277683:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949028.277690:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949028.277693:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:1.0:1784949028.277697:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439d80 00000100:00000200:0.0:1784949028.277698:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:1.0:1784949028.277704:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8b91 [1] + 224 00000100:00000010:0.0:1784949028.277708:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d003ffea. 00000400:00000200:1.0:1784949028.277710:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784949028.277716:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000d003ffea x1871651289669248 msgsize 224 00000400:00000200:1.0:1784949028.277717:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949028.277738:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784949028.277744:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:1.0:1784949028.277746:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949028.277760:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949028.277762:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784949028.277771:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949028.277778:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949028.277784:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949028.277792:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784949028.277795:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000100:00100000:2.0:1784949028.277796:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289669248 00000400:00000010:1.0:1784949028.277800:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 02000000:00000001:2.0:1784949028.277800:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949028.277803:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949028.277806:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949028.277808:0:342756:0:(events.c:386:reply_out_callback()) Process entered 02000000:00000001:2.0:1784949028.277809:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949028.277812:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00100000:2.0:1784949028.277813:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289669248 00000020:00000001:2.0:1784949028.277817:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949028.277819:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:1.0:1784949028.277822:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949028.277822:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949028.277822:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1784949028.277824:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:3.0:1784949028.277828:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949028.277829:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:1.0:1784949028.277832:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:2.0:1784949028.277834:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000001:3.0:1784949028.277836:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000001:1.0:1784949028.277840:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:2.0:1784949028.277841:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000001:3.0:1784949028.277841:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1784949028.277846:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949028.277846:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000400:00000010:3.0:1784949028.277847:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001ee400ea. 00000020:00000001:1.0:1784949028.277854:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784949028.277854:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949028.277857:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000020:00000010:2.0:1784949028.277859:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00000040:1.0:1784949028.277861:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009c11a764 x1871651289668992/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:689/0 lens 224/224 e 0 to 0 dl 1784949039 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949028.277863:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000020:00000010:2.0:1784949028.277866:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e5260462. 00000100:00000001:3.0:1784949028.277870:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:2.0:1784949028.277873:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ce523b1b. 00000100:00000200:3.0:1784949028.277878:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949028.277880:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00100000:1.0:1784949028.277885:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009c11a764 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289668992:12345-0@lo:400: Request processed in 415us (1057us total) trans 0 rc 0/0 00000100:00000040:2.0:1784949028.277890:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949028.277896:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949028.277899:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:3.0:1784949028.277928:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949028.277931:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949028.277939:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 759 00000100:00000001:2.0:1784949028.277940:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949028.277944:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949028.277948:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949028.277949:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949028.277949:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949028.277953:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949028.277954:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949028.277956:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949028.277960:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949028.277960:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949028.277966:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:1.0:1784949028.277967:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003cf4ade9. 00000100:00000001:2.0:1784949028.277968:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949028.277969:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949028.277972:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949028.277974:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949028.277977:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949028.277980:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:2.0:1784949028.277981:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 735 02000000:00000001:3.0:1784949028.277981:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949028.277985:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000010:1.0:1784949028.277987:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000f7a7941e. 00000100:00000040:2.0:1784949028.277989:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949028.277990:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949028.277991:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=50 v=5 (5 5 5 5) 00000100:00000001:2.0:1784949028.277994:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554964480 : -115544154587136 : ffff96e9c7b5c000) 00000100:00000001:0.0:1784949028.277996:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949028.277998:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000040:2.0:1784949028.278004:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d003ffea x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:689/0 lens 224/0 e 0 to 0 dl 1784949039 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:3.0:1784949028.278004:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=50 v=5 (1 1 1 1) 02000000:00000001:1.0:1784949028.278008:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949028.278014:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000e5630c1a. 00000100:00000001:3.0:1784949028.278015:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949028.278018:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949028.278022:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949028.278026:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.278026:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:3.0:1784949028.278026:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949028.278030:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949028.278031:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000200:1.0:1784949028.278033:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949028.278034:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.278035:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949028.278040:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000400:00000010:1.0:1784949028.278042:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002f6a97f7. 00000100:00000040:3.0:1784949028.278044:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949028.278049:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000043dfa704. 00000100:00100000:2.0:1784949028.278049:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d003ffea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289669248:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784949028.278059:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289669248 00000020:00000001:2.0:1784949028.278080:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949028.278084:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1784949028.278086:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949028.278091:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949028.278094:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1784949028.278096:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784949028.278102:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949028.278103:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949028.278113:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949028.278116:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949028.278118:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949028.278120:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949028.278123:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949028.278127:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949028.278132:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949028.278137:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949028.278141:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949028.278149:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000052be9ded. 00000100:00100000:3.0:1784949028.278149:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289668992:0@lo:400: 02000000:00000001:2.0:1784949028.278158:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949028.278160:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949028.278162:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949028.278167:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949028.278168:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949028.278173:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289669248 00000100:00000040:3.0:1784949028.278174:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000046cb422a x1871651289668992/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949028.278179:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949028.278187:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d003ffea x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:689/0 lens 224/224 e 0 to 0 dl 1784949039 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949028.278192:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949028.278197:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949028.278203:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 00010000:00000001:2.0:1784949028.278204:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949028.278209:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949028.278210:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00001000:2.0:1784949028.278212:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=45 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949028.278214:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949028.278219:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949028.278222:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1784949028.278224:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:2.0:1784949028.278229:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000010:3.0:1784949028.278231:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 00000100:00000001:2.0:1784949028.278234:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949028.278240:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949028.278240:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949028.278241:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949028.278248:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949028.278248:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1784949028.278255:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000010:3.0:1784949028.278255:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000046cb422a. 00000400:00000010:2.0:1784949028.278262:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000100:00000001:3.0:1784949028.278265:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000200:2.0:1784949028.278268:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289669248, offset 224 00000100:00000001:3.0:1784949028.278272:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949028.278280:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949028.278281:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.278291:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1784949028.278292:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949028.278298:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949028.278304:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3520:3520:247:4294967295] 192.168.202.137@tcp LPNI seq info [2273:2273:8:4294967295] 00000100:00000001:3.0:1784949028.278307:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949028.278312:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949028.278322:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949028.278329:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949028.278334:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005523afda. 00000800:00000200:2.0:1784949028.278339:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949028.278345:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949028.278348:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005523afda type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949028.278363:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949028.278366:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949028.278369:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949028.278371:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949028.278373:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949028.278376:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d003ffea x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:689/0 lens 224/224 e 0 to 0 dl 1784949039 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949028.278388:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d003ffea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289669248:12345-192.168.202.137@tcp:400: Request processed in 361us (670us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949028.278397:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 735 00000100:00000040:2.0:1784949028.278401:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949028.278403:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949028.278406:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949028.278409:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ce523b1b. 00000020:00000010:2.0:1784949028.278412:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949028.278415:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e5260462. 00000020:00000040:2.0:1784949028.278422:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949028.278425:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949028.278500:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949028.278506:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949028.278508:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949028.278509:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949028.278513:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949028.278515:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005523afda. 00000400:00000200:2.0:1784949028.278521:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949028.278524:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949028.278529:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6439e80 00000400:00000200:3.0:1784949028.278535:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949028.278540:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8b99 [1] + 224 00000800:00000001:2.0:1784949028.278545:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949028.278545:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:3.0:1784949028.278549:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:3.0:1784949028.278554:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784949028.278555:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.278556:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949028.278559:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949028.278564:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949028.278568:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949028.278571:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949028.278578:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949028.278595:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949028.278600:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949028.278603:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.278609:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949028.278613:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949028.278616:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949028.278618:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949028.278619:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000400:00000200:3.0:1784949028.278623:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:3.0:1784949028.278624:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000100:00000001:3.0:1784949028.278626:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949028.278630:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949028.278636:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949028.278646:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949028.278647:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.278648:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949028.278650:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949028.278651:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949028.278652:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949028.278653:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949028.278657:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=45 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949028.278659:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=45 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949028.278661:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949028.278662:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949028.278664:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949028.278666:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949028.278671:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949028.278673:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949028.278674:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949028.278675:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949028.278676:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949028.278678:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949028.278683:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949028.278684:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949028.278685:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949028.278687:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949028.278692:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289669248:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949028.278695:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.278696:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949028.278698:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289669248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949044 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949028.278706:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949028.278708:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949028.278709:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000015713dee. 02000000:00000001:3.0:1784949028.278712:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949028.278713:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949028.278713:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949028.278715:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949028.278716:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:3.0:1784949028.278736:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949028.278737:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949028.278738:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:3.0:1784949028.278741:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949028.278742:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.278743:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949028.278746:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949028.278747:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949028.278748:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949028.278750:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949028.319967:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305967659, at 4305967660 00000100:00000040:3.0:1784949028.319976:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305967659, at 4305967660 00000100:00000010:2.0:1784949028.319977:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000010:3.0:1784949028.319986:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:0.0:1784949028.319994:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305967659, at 4305967660 00000100:00000040:1.0:1784949028.319999:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305967659, at 4305967660 00000100:00000010:0.0:1784949028.320002:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000010:1.0:1784949028.320012:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000800:00000100:2.0:1784949028.468023:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.8@tcp 00000800:00000010:2.0:1784949028.468029:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f495c80e. 00000400:00000200:2.0:1784949028.468038:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949028.468059:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949028.468068:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11301 health :900 00000400:00000200:2.0:1784949028.468075:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949028.468081:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:2.0:1784949028.468086:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000800:00000100:2.0:1784949028.468116:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000800:00000010:2.0:1784949028.468120:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cf155122. 00000400:00000200:2.0:1784949028.468124:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949028.468130:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949028.468139:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949028.468145:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:2.0:1784949028.468152:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:1.0:1784949028.468165:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949028.468178:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:1.0:1784949028.468187:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949028.468189:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949028.468193:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:1.0:1784949028.468195:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000400:00000200:1.0:1784949028.468201:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949028.468207:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:1.0:1784949028.468211:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f1904ca. 00000400:00000200:1.0:1784949028.468217:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-110 00000400:00000200:1.0:1784949028.468220:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -110 00000400:00000200:1.0:1784949028.468225:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000400:00000200:1.0:1784949028.468229:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000001:1.0:1784949028.468232:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949028.468234:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949028.468236:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000400:00000010:1.0:1784949028.468238:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000400:00000200:1.0:1784949028.468240:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949028.468243:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:1.0:1784949028.468245:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000d531934e. 00000400:00000200:1.0:1784949028.468249:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:1.0:1784949028.468251:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:1.0:1784949028.468255:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00000800:00000001:0.0:1784949028.687532:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949028.687553:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949028.687559:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949028.687560:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949029.108018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949029.108028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949029.108037:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949029.108039:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949029.108173:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949029.108180:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949029.108185:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949029.300067:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949029.300076:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949029.300092:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949028/real 0] req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949029 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949029.300109:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.300113:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949029.300116:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949029.300122:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949029 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949029.300132:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.300142:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949029.300144:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.300148:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949029.300153:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949029.300156:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949029.300161:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949029 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949029.300170:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949029.300175:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.300177:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949029.300179:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.300181:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949029.300184:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949029.300188:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949029.300219:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784949029.300224:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949029.300231:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949029 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949029.300247:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289669120:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949029.300259:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.300265:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949029.300271:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006d868603 x1871651289669120/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949029 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949029.300286:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784949029.300291:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949029.300295:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784949029.300298:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949029.300304:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000008142e1f1. 02000000:00000001:0.0:1784949029.300313:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949029.300318:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949029.300321:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006d868603. 00000100:00000001:0.0:1784949029.300343:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949029.300346:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.300350:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.300358:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949029.300361:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.300366:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949029.300369:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949029.300391:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949029.300402:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949029.300411:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000040:3.0:1784949029.300417:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949029.300438:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949029.300441:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949029.300445:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949029.300452:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:3.0:1784949029.300458:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949029.300463:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949029.300472:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949029.300474:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949029.300479:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_01+1] req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949029.300507:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949029.300510:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949029.300514:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784949029.300517:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784949029.300519:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784949029.300524:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784949029.300529:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11275 00000100:00000001:1.0:1784949029.300534:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949029.300540:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000400:00000200:3.0:1784949029.300542:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:1.0:1784949029.300544:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949029.300553:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1784949029.300553:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11300 00000100:00000001:1.0:1784949029.300557:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949029.300561:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949029.300563:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:1.0:1784949029.300564:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1784949029.300568:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1784949029.300569:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949029.300575:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949029.300576:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000040:1.0:1784949029.300578:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949029.300580:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.300585:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949029.300592:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949029.300594:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949029.300598:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.300599:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949029.300602:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949029.300606:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949029.300606:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949029.300611:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:1.0:1784949029.300612:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949029.300617:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289669376:0@lo:400: 00000020:00000001:3.0:1784949029.300618:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949029.300623:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1784949029.300624:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:1.0:1784949029.300628:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1784949029.300629:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784949029.300631:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949029.300637:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784949029.300638:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949029.300643:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 00000100:00000040:3.0:1784949029.300645:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000001:1.0:1784949029.300647:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.300651:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1784949029.300655:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000057b2318d. 00000100:00000040:3.0:1784949029.300656:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949029.300661:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000010:1.0:1784949029.300664:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000020:00000040:3.0:1784949029.300665:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000200:1.0:1784949029.300672:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289669376, portal 25 00000100:00080000:3.0:1784949029.300673:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000001:1.0:1784949029.300677:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949029.300679:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000100:00080000:3.0:1784949029.300682:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000040:1.0:1784949029.300687:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949029.300688:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949029.300694:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1784949029.300699:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949029.300702:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784949029.300702:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784949029.300706:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 02000000:00000001:3.0:1784949029.300708:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949029.300726:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289669376, offset 0 00000100:00000001:3.0:1784949029.300732:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1784949029.300748:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949029.300752:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000010:3.0:1784949029.300756:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046cb422a. 00000020:00000040:3.0:1784949029.300761:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784949029.300765:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784949029.300767:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949029.300773:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949029.300774:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949029.300778:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949029.300782:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949029.300788:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6439f00 02000000:00000010:3.0:1784949029.300788:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000579609a4. 00000100:00000001:3.0:1784949029.300795:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949029.300801:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8209 [2] + 0 00000100:00100000:3.0:1784949029.300805:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1784949029.300810:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a23805e4. 00000400:00000200:1.0:1784949029.300818:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000e9d273c9 00000100:00000001:3.0:1784949029.300821:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949029.300824:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949029.300827:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784949029.300832:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046cb422a] to pc [ptlrpcd_rcv+-1] req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949029.300835:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949029.300842:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e9d273c9 00000400:00000010:1.0:1784949029.300845:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000e9d273c9. 00000100:00000001:1.0:1784949029.300852:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949029.300854:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784949029.300855:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949029.300857:0:337119:0:(events.c:337:request_in_callback()) incoming req@000000001e461ac8 x1871651289669376 msgsize 224 00000100:00000040:3.0:1784949029.300861:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1784949029.300869:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784949029.300872:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000040:1.0:1784949029.300873:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949029.300875:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949029.300879:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:3.0:1784949029.300879:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001445c4ff. 00000020:00000040:3.0:1784949029.300885:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949029.300887:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949029.300890:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949029.300894:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1784949029.300896:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 02000000:00000001:3.0:1784949029.300897:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949029.300900:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000100:00000001:2.0:1784949029.300922:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949029.300929:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949029.300931:0:337119:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:3.0:1784949029.300936:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000200:1.0:1784949029.300942:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949029.300943:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949029.300945:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.300946:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949029.300947:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949029.300970:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289669376 02000000:00000001:2.0:1784949029.300973:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949029.300980:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949029.300981:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1784949029.300982:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949029.300991:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:2.0:1784949029.300991:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949029.301007:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289669504:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784949029.301010:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949029.301022:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949029.301023:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949029.301034:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949029.301039:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.301043:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949029.301049:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1784949029.301057:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949029.301069:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289669376 00000100:00000001:1.0:1784949029.301075:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949029.301077:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949029.301079:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949029.301082:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:0.0:1784949029.301085:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949029.301085:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949029.301091:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949029.301095:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949029.301097:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000040:3.0:1784949029.301102:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001445c4ff] to pc [ptlrpcd_00_02+2] req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949029.301107:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949029.301108:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305968683 00000020:00000001:0.0:1784949029.301116:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784949029.301122:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784949029.301126:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949029.301135:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:0.0:1784949029.301137:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:0.0:1784949029.301148:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949029.301149:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305968683 00000100:00000001:3.0:1784949029.301151:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949029.301157:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000010:1.0:1784949029.301165:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000040:3.0:1784949029.301166:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11306) 00000020:00000010:0.0:1784949029.301169:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:1.0:1784949029.301175:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:0.0:1784949029.301182:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c8206cde. 00000100:00000001:1.0:1784949029.301185:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305968683 00000100:00000001:3.0:1784949029.301195:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784949029.301198:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000020:00000010:0.0:1784949029.301200:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949029.301202:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949029.301207:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949029.301209:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949029.301223:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949029.301224:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305968683 00000100:00000040:3.0:1784949029.301225:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949029.301234:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000010:2.0:1784949029.301235:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000c4a1e5eb. 00000100:00000010:1.0:1784949029.301238:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:0.0:1784949029.301240:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949029.301249:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949029.301251:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949029.301252:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784949029.301258:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949029.301262:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949029.301264:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949029.301266:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949029.301275:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.301277:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949029.301278:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949029.301281:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949029.301284:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949029.301288:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949029.301297:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.301302:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784949029.301306:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000001445c4ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289669632:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949029.301307:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949029.301316:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 760 00000100:00000001:3.0:1784949029.301318:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949029.301323:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 02000000:00000001:3.0:1784949029.301323:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949029.301327:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.301328:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180528208 : -115539529023408 : ffff96eadb6a4050) 02000000:00000001:3.0:1784949029.301332:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784949029.301337:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001e461ac8 x1871651289669376/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:690/0 lens 224/0 e 0 to 0 dl 1784949040 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949029.301340:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000062d113c0. 02000000:00000010:3.0:1784949029.301343:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 02000000:00000001:3.0:1784949029.301351:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949029.301357:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000100:00000001:0.0:1784949029.301361:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:3.0:1784949029.301361:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000df521cee. 00000100:00000200:2.0:1784949029.301366:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289669504, portal 4 00000020:00000010:0.0:1784949029.301368:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000400:00000010:3.0:1784949029.301372:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000100:00000001:2.0:1784949029.301375:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949029.301378:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949029.301379:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554953472 : -115544154598144 : ffff96e9c7b59500) 00000100:00000200:3.0:1784949029.301381:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289669632, portal 10 00000100:00000001:1.0:1784949029.301383:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784949029.301394:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:1.0:1784949029.301394:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.301395:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949029.301402:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554962432 : -115544154589184 : ffff96e9c7b5b800) 00000100:00000040:2.0:1784949029.301404:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949029.301428:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001e461ac8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289669376:12345-0@lo:400: 00000100:00000040:3.0:1784949029.301434:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949029.301439:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289669376 00000020:00000001:0.0:1784949029.301445:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949029.301448:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1784949029.301451:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949029.301453:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1784949029.301455:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949029.301456:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949029.301459:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:3.0:1784949029.301461:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949029.301462:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000020:00000001:0.0:1784949029.301463:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000010:3.0:1784949029.301465:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000100:00000200:2.0:1784949029.301468:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289669504, offset 0 00000020:00000001:0.0:1784949029.301470:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000200:3.0:1784949029.301470:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289669632, offset 0 00000020:00000001:0.0:1784949029.301473:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949029.301477:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949029.301478:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949029.301483:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949029.301486:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949029.301488:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949029.301489:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949029.301490:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949029.301495:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:2.0:1784949029.301496:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784949029.301499:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:2.0:1784949029.301503:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000010:0.0:1784949029.301505:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006296bcc7. 00000400:00000200:2.0:1784949029.301507:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784949029.301512:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949029.301513:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784949029.301518:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949029.301519:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3521:3521:249:4294967295] 192.168.202.137@tcp LPNI seq info [2274:2274:8:4294967295] 00000020:00000001:0.0:1784949029.301523:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949029.301527:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289669376 00010000:00000001:0.0:1784949029.301533:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:2.0:1784949029.301535:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00010000:00000200:0.0:1784949029.301540:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001e461ac8 x1871651289669376/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:690/0 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949029.301543:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949029.301546:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000800:00000010:2.0:1784949029.301551:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e1f7df0e. 00000400:00000200:3.0:1784949029.301552:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:0.0:1784949029.301554:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949029.301558:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949029.301558:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00001000:0.0:1784949029.301562:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=51 v=5 (1 1 1 1) 00000800:00000200:2.0:1784949029.301563:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784949029.301564:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949029.301569:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1784949029.301571:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:0.0:1784949029.301572:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:3.0:1784949029.301574:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3522:3522:248:4294967295] 192.168.202.137@tcp LPNI seq info [2275:2275:7:4294967295] 00000100:00000001:0.0:1784949029.301575:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000200:2.0:1784949029.301580:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 02000000:00000001:0.0:1784949029.301581:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949029.301585:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949029.301588:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e1f7df0e type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1784949029.301589:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949029.301589:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:0.0:1784949029.301594:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949029.301600:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000800:00000200:3.0:1784949029.301601:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000200:0.0:1784949029.301603:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289669376, offset 224 00000800:00000010:3.0:1784949029.301605:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000400:00000200:0.0:1784949029.301608:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000200:3.0:1784949029.301610:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949029.301615:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949029.301615:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949029.301616:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949029.301622:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000040:2.0:1784949029.301623:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000400:00000200:0.0:1784949029.301628:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6439f00 00000100:00000001:2.0:1784949029.301630:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305968683 00000100:00000001:2.0:1784949029.301636:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305968683 00000100:00000001:3.0:1784949029.301638:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949029.301640:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305968683 00000400:00000200:0.0:1784949029.301642:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8bad [1] + 224 00000100:00000001:2.0:1784949029.301643:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305968683 00000100:00000040:3.0:1784949029.301644:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2136 usecs (3 jiffies) 00000100:00000001:2.0:1784949029.301648:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305968683 00000100:00000001:2.0:1784949029.301651:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305968683 00000100:00000001:3.0:1784949029.301651:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305968644 00000400:00000200:0.0:1784949029.301653:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949029.301656:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305968683 00000100:00000001:3.0:1784949029.301658:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305968683 00000100:00000001:2.0:1784949029.301659:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305968683 00000400:00000200:0.0:1784949029.301662:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949029.301664:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305968644 00000100:00000001:2.0:1784949029.301665:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949029.301669:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949029.301670:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305968683 00000100:00000001:2.0:1784949029.301671:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949029.301675:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949029.301678:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.301678:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305968644 00000100:00000001:2.0:1784949029.301684:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.301684:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305968683 00000100:00000001:3.0:1784949029.301689:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305968644 00000100:00000001:2.0:1784949029.301692:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949029.301693:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949029.301693:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305968683 00000100:00000001:2.0:1784949029.301695:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1784949029.301697:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949029.301702:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949029.301703:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949029.301706:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949029.301708:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949029.301728:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949029.301730:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.301731:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000010:1.0:1784949029.301735:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e1f7df0e. 00000100:00000001:3.0:1784949029.301737:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1784949029.301738:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:0.0:1784949029.301741:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000001:3.0:1784949029.301741:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:1.0:1784949029.301743:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949029.301745:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949029.301745:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949029.301748:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.301749:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949029.301749:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949029.301754:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949029.301755:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.301755:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.301756:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.301764:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949029.301765:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:1.0:1784949029.301766:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00010000:00000001:0.0:1784949029.301768:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:1.0:1784949029.301771:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000020:00000001:0.0:1784949029.301773:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949029.301775:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949029.301778:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784949029.301779:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:1.0:1784949029.301785:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949029.301787:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949029.301788:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001e461ac8 x1871651289669376/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:690/0 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949029.301794:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949029.301799:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949029.301815:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949029.301816:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949029.301823:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949054 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949029.301829:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001e461ac8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289669376:12345-0@lo:400: Request processed in 465us (966us total) trans 0 rc 0/0 00000400:00000200:2.0:1784949029.301832:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6439f80 00000400:00000100:2.0:1784949029.301847:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289669504 offset 0 length 520: 4 00000100:00000001:1.0:1784949029.301849:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949029.301854:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000001619d5fe is in inconsistent state, don't perform health checking (-2, 0) 00000100:00100000:0.0:1784949029.301855:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 760 00000100:00000001:1.0:1784949029.301856:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784949029.301860:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000040:0.0:1784949029.301865:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949029.301870:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000001:2.0:1784949029.301871:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949029.301872:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949029.301878:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c8206cde. 00000100:00000001:3.0:1784949029.301882:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784949029.301883:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949029.301885:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:0.0:1784949029.301886:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000400:00000001:3.0:1784949029.301891:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:0.0:1784949029.301892:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000001:3.0:1784949029.301893:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949029.301895:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000057b2318d. 02000000:00000001:0.0:1784949029.301897:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000400:00000200:3.0:1784949029.301899:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 02000000:00000010:0.0:1784949029.301900:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006296bcc7. 00000400:00000010:3.0:1784949029.301902:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 02000000:00000001:0.0:1784949029.301903:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949029.301931:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949029.301933:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949029.301936:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949029.301938:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:0.0:1784949029.301944:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f301fed7. 00000100:00000200:3.0:1784949029.301946:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949029.301949:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000feb5569e. 00000100:00000001:3.0:1784949029.301955:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949029.301956:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.301975:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949029.301980:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949029.301983:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:1.0:1784949029.301988:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949029.301989:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949029.301998:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 02000000:00000001:3.0:1784949029.302001:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949029.302006:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949029.302010:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:3.0:1784949029.302013:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=51 v=5 (5 5 5 5) 00000800:00000001:0.0:1784949029.302017:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949029.302020:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784949029.302024:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784949029.302026:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=51 v=5 (1 1 1 1) 00000800:00000001:0.0:1784949029.302027:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949029.302030:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000800:00000001:2.0:1784949029.302034:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784949029.302038:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000800:00000001:0.0:1784949029.302039:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.302044:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000800:00000001:2.0:1784949029.302045:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949029.302048:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949029.302050:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949029.302061:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949029.302062:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000200:2.0:1784949029.302063:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:3.0:1784949029.302074:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.302087:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949029.302090:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a000 00000100:00000040:3.0:1784949029.302106:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949029.302111:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949029.302120:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 165904 00000100:00000040:1.0:1784949029.302122:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949029.302148:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949029.302160:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949029.302163:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.302166:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949029.302171:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1784949029.302172:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.302173:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949029.302185:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784949029.302190:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949029.302209:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:3.0:1784949029.302211:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289669376:0@lo:400: 00000100:00000001:3.0:1784949029.302227:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949029.302231:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949029.302231:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:2.0:1784949029.302240:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:3.0:1784949029.302241:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289669376/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784949029.302256:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005885bb8b. 00000100:00000001:3.0:1784949029.302259:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949029.302264:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1784949029.302268:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000005885bb8b x1871651289669632 msgsize 224 02000000:00000010:3.0:1784949029.302269:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 02000000:00000001:3.0:1784949029.302283:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949029.302288:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00100000:2.0:1784949029.302289:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000040:3.0:1784949029.302291:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949029.302301:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949029.302303:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:3.0:1784949029.302307:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949029.302308:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949029.302310:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:3.0:1784949029.302314:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949029.302316:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.302318:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.302324:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949029.302327:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949029.302328:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.302337:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949029.302342:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949029.302345:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949029.302355:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.302398:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949029.302405:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289669632 02000000:00000001:0.0:1784949029.302409:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949029.302412:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949029.302414:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949029.302419:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949029.302422:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289669632 00000020:00000001:0.0:1784949029.302425:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949029.302443:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949029.302444:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949029.302446:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949029.302448:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949029.302450:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949029.302453:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949029.302456:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:0.0:1784949029.302457:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949029.302459:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000020:00000010:0.0:1784949029.302462:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000020:00000010:0.0:1784949029.302465:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c8206cde. 00000020:00000010:0.0:1784949029.302467:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:0.0:1784949029.302472:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949029.302473:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949029.302474:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949029.302476:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.302480:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949029.302482:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949029.302483:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.302492:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949029.302499:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949029.302500:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949029.302503:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 736 00000100:00000040:0.0:1784949029.302505:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949029.302506:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562024832 : -115544147526784 : ffff96e9c8217b80) 00000100:00000040:0.0:1784949029.302510:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005885bb8b x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:690/0 lens 224/0 e 0 to 0 dl 1784949040 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949029.302516:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949029.302518:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:0.0:1784949029.302519:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949029.302520:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00100000:0.0:1784949029.302523:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005885bb8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289669632:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949029.302526:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289669632 00000020:00000001:0.0:1784949029.302527:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949029.302529:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949029.302530:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949029.302531:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949029.302533:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949029.302534:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949029.302536:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949029.302538:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949029.302539:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949029.302540:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949029.302543:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949029.302546:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949029.302547:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949029.302549:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006296bcc7. 02000000:00000001:0.0:1784949029.302551:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.302553:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949029.302554:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949029.302556:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289669632 00010000:00000001:0.0:1784949029.302558:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949029.302561:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005885bb8b x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:690/0 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949029.302567:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949029.302569:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949029.302570:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=46 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949029.302572:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949029.302574:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949029.302576:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949029.302578:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949029.302579:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949029.302580:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949029.302582:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949029.302583:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000100:00000200:0.0:1784949029.302586:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289669632, offset 224 00000400:00000200:0.0:1784949029.302589:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949029.302594:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949029.302597:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3523:3523:249:4294967295] 192.168.202.137@tcp LPNI seq info [2276:2276:8:4294967295] 00000400:00000200:0.0:1784949029.302604:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949029.302607:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949029.302610:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d1b12746. 00000800:00000200:0.0:1784949029.302613:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949029.302617:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949029.302619:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000d1b12746 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949029.302629:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949029.302631:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949029.302633:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949029.302634:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949029.302635:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949029.302637:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005885bb8b x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:690/0 lens 224/224 e 0 to 0 dl 1784949040 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949029.302645:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005885bb8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289669632:12345-192.168.202.137@tcp:400: Request processed in 127us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949029.302652:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 736 00000100:00000040:0.0:1784949029.302654:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949029.302655:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949029.302657:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949029.302659:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c8206cde. 00000020:00000010:0.0:1784949029.302662:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949029.302664:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000020:00000040:0.0:1784949029.302666:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949029.302668:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949029.302698:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949029.302701:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d1b12746. 00000400:00000200:2.0:1784949029.302705:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949029.302725:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949029.302729:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:2.0:1784949029.302730:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000001:2.0:1784949029.302733:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949029.302735:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949029.302736:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949029.302747:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949029.302751:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949029.302754:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949029.302761:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949029.302771:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a000 00000400:00000200:1.0:1784949029.302775:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8bb9 [1] + 224 00000800:00000001:1.0:1784949029.302779:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949029.302783:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949029.302784:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949029.302787:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949029.302791:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949029.302795:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949029.302806:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949029.302825:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949029.302833:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949029.302836:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.302856:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949029.302859:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949029.302863:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949029.302864:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949029.302865:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000400:00000200:3.0:1784949029.302869:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:3.0:1784949029.302870:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000100:00000001:3.0:1784949029.302872:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949029.302876:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949029.302883:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949029.302888:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949029.302889:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.302890:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949029.302891:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949029.302892:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949029.302893:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949029.302895:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949029.302898:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=46 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949029.302900:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=46 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949029.302902:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949029.302902:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949029.302921:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949029.302924:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949029.302930:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.302932:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949029.302950:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949029.302953:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949029.302955:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949029.302959:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949029.302967:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949029.302969:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949029.302971:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949029.302974:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949029.302982:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000001445c4ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289669632:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949029.302987:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.302991:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949029.302993:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000001445c4ff x1871651289669632/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949029.303001:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949029.303004:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949029.303006:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:3.0:1784949029.303010:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949029.303011:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949029.303013:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949029.303016:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949029.303017:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 02000000:00000001:3.0:1784949029.303020:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949029.303022:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949029.303023:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000001445c4ff. 00000100:00000001:3.0:1784949029.303027:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949029.303028:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.303031:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949029.303035:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949029.303037:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949029.303039:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949029.303041:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949029.343967:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305968683, at 4305968684 00000100:00000010:2.0:1784949029.343975:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:0.0:1784949029.343991:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305968683, at 4305968684 00000100:00000040:3.0:1784949029.343991:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305968683, at 4305968684 00000100:00000010:0.0:1784949029.344001:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:3.0:1784949029.344001:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:1.0:1784949029.344084:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305968683, at 4305968684 00000100:00000010:1.0:1784949029.344114:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000800:00000100:2.0:1784949029.491980:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:2.0:1784949029.491984:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b9b32818. 00000400:00000200:2.0:1784949029.491989:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949029.491994:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949029.491999:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949029.492002:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:2.0:1784949029.492006:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:1.0:1784949029.492118:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949029.492133:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:1.0:1784949029.492144:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949029.492147:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949029.492151:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:1.0:1784949029.492155:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000400:00000200:1.0:1784949029.492163:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949029.492175:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:1.0:1784949029.492182:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004f865a97. 00000400:00000200:1.0:1784949029.492190:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000400:00000200:1.0:1784949029.492200:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000400:00000200:3.0:1784949029.492202:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:1.0:1784949029.492208:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000400:00000010:3.0:1784949029.492216:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000008d3e21a4. 00000400:00000010:3.0:1784949029.492227:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:3.0:1784949029.492234:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000400:00000010:3.0:1784949029.492244:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000061508776. 00000400:00000200:3.0:1784949029.492251:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949029.492264:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949029.492270:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949029.492274:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3524:3524:250:4294967295] 192.168.202.137@tcp LPNI seq info [2277:2277:8:4294967295] 00000400:00000200:3.0:1784949029.492281:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949029.492288:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949029.492295:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949029.492301:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000800:00000200:3.0:1784949029.492307:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949029.492314:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949029.492318:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949029.492340:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11303 health :900 00000800:00000200:2.0:1784949029.492440:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949029.492444:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:2.0:1784949029.492447:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949029.492452:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949029.492456:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784949029.492457:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949029.492460:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784949029.492468:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949029.492472:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949029.492479:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949029.492485:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949029.492494:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949029.492501:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949029.492507:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949029.492510:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3525:3525:250:4294967295] 192.168.202.137@tcp LPNI seq info [2278:2278:8:4294967295] 00000400:00000200:0.0:1784949029.492517:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949029.492521:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949029.492523:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000086e4d21. 00000800:00000200:0.0:1784949029.492527:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949029.492530:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949029.492532:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000086e4d21 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:0.0:1784949029.492585:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949029.492587:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000086e4d21. 00000400:00000200:0.0:1784949029.492590:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949029.492593:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949029.492599:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949029.492603:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949029.492604:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949029.492606:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949029.492608:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949029.492614:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8bd1 00000800:00000001:0.0:1784949029.492618:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949029.492622:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949029.492625:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949029.492628:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949029.492633:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:0.0:1784949029.492635:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000400:00000200:0.0:1784949029.492638:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949029.492641:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000008d3e21a4. 00000400:00000010:0.0:1784949029.492644:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000800:00000001:0.0:1784949029.492647:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949029.492650:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949030.131983:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949030.131989:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949030.131995:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949030.131996:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949030.132032:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949030.132036:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949030.132039:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949031.156008:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949031.156015:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949031.156021:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949031.156024:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949031.156026:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949031.156029:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949031.156031:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949031.156033:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949031.156035:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949031.156037:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949031.156141:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949031.156149:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949031.156154:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1784949031.539989:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.27@tcp 00000800:00000010:2.0:1784949031.539997:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cb55bcd8. 00000400:00000200:2.0:1784949031.540006:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949031.540014:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.27@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949031.540021:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11304 health :900 00000400:00000200:2.0:1784949031.540025:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949031.540028:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.27@tcp: 1 00000400:00000200:2.0:1784949031.540031:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.27@tcp: -114 00000800:00000100:2.0:1784949031.540057:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000800:00000010:2.0:1784949031.540061:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ca1124f5. 00000400:00000200:2.0:1784949031.540066:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949031.540072:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949031.540079:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949031.540085:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:1.0:1784949031.540087:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949031.540091:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:1.0:1784949031.540107:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.27@tcp(0000000041892d66) state 0x840 00000400:00000001:1.0:1784949031.540118:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.540121:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.540126:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:1.0:1784949031.540129:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000400:00000200:1.0:1784949031.540136:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.540141:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.27@tcp: -114 00000400:00000010:1.0:1784949031.540146:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef171f31. 00000400:00000200:1.0:1784949031.540157:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.27@tcp:-110 00000400:00000200:1.0:1784949031.540160:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.27@tcp(0000000041892d66) state 0x2040 rc -110 00000400:00000200:1.0:1784949031.540163:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.27@tcp 00000400:00000200:1.0:1784949031.540184:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:1.0:1784949031.540187:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.540188:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.540190:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:1.0:1784949031.540191:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:1.0:1784949031.540193:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.540195:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:1.0:1784949031.540197:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000072ce2c8. 00000400:00000200:1.0:1784949031.540200:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:1.0:1784949031.540202:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:1.0:1784949031.540204:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:3.0:1784949031.540247:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.27@tcp NID 111.222.142.27@tcp: -110. discovery complete 00000400:00000200:3.0:1784949031.540255:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.142.27@tcp primary NID 111.222.142.27@tcp rc -110 00000100:00000010:3.0:1784949031.540278:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000b4985bb9. 00000100:00000001:3.0:1784949031.540309:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784949031.540311:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000b4985bb9 refcount 1 to 111.222.142.27@tcp 00010000:00000010:3.0:1784949031.540317:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 000000007024d30a. 00010000:00080000:3.0:1784949031.540362:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.142.27@tcp at tail 00010000:00000001:3.0:1784949031.540365:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949031.540370:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949031.540396:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 000000008142e1f1. 00000100:00000400:3.0:1784949031.540401:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.173.28@tcp: rc = -75 00000100:00000040:3.0:1784949031.540404:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.173.28@tcp cnt=32 00000400:00000200:3.0:1784949031.540407:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784949031.540410:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784949031.540412:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784949031.540414:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784949031.540416:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784949031.540424:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784949031.540429:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540431:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540434:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784949031.540435:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784949031.540439:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949031.540455:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784949031.540457:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.540458:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540461:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540463:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784949031.540464:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784949031.540467:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949031.540470:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784949031.540472:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.540474:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540476:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540478:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784949031.540480:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784949031.540483:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949031.540490:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784949031.540492:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949031.540500:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949031.540501:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540508:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540509:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784949031.540511:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:1.0:1784949031.540515:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784949031.540517:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949031.540525:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:3.0:1784949031.540526:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.540527:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:1.0:1784949031.540528:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000043d62367. 00000400:00000010:1.0:1784949031.540536:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000400:00000200:3.0:1784949031.540541:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540544:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000010:1.0:1784949031.540546:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bfef55e5. 00000400:00000200:3.0:1784949031.540546:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784949031.540556:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.540561:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949031.540565:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784949031.540568:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.540572:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540574:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540576:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:3.0:1784949031.540578:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000200:3.0:1784949031.540582:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.540587:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:1.0:1784949031.540593:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.540599:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.540603:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.540606:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:1.0:1784949031.540611:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3526:3526:252:4294967295] 111.222.173.1@tcp LPNI seq info [50:50:8:4294967295] 00000400:00000200:1.0:1784949031.540618:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:1.0:1784949031.540625:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.540632:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949031.540635:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000400:00000200:3.0:1784949031.540639:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:1.0:1784949031.540640:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000400:00000200:3.0:1784949031.540641:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540658:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540664:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000800:00000200:1.0:1784949031.540665:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949031.540671:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:3.0:1784949031.540678:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000800:00000200:1.1:1784949031.540679:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949031.540684:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784949031.540687:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.540688:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949031.540717:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:1.0:1784949031.540720:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:3.0:1784949031.540816:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.540820:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784949031.540823:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:1.0:1784949031.540825:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784949031.540830:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949031.540832:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000035b54a67. 00000400:00000010:1.0:1784949031.540837:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000400:00000200:3.0:1784949031.540838:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:3.0:1784949031.540843:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949031.540846:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001f662712. 00000400:00000200:3.0:1784949031.540846:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540856:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:1.0:1784949031.540857:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949031.540863:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784949031.540867:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000200:1.0:1784949031.540871:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784949031.540875:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.540879:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.540884:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.540886:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.540899:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:1.0:1784949031.540902:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3527:3527:251:4294967295] 111.222.142.1@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:1.0:1784949031.540936:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:1.0:1784949031.540942:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.540949:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000800:00000010:1.0:1784949031.540955:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000400:00000200:3.0:1784949031.540961:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000800:00000200:1.0:1784949031.540967:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949031.540968:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.0:1784949031.540973:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 9 secs later) 00000400:00000200:3.0:1784949031.540975:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.540985:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000800:00000200:1.1:1784949031.540986:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000800:00000200:1.1:1784949031.540989:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 9 secs later) 00000800:00000100:1.0:1784949031.540994:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.1@tcp 00000800:00000010:1.0:1784949031.540997:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000400:00000200:1.0:1784949031.541002:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.541006:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949031.541074:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:1.0:1784949031.541076:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949031.541078:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:1.0:1784949031.541084:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:3.0:1784949031.541088:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541092:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:1.0:1784949031.541105:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:3.0:1784949031.541110:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000200:1.0:1784949031.541111:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 rc 0 00000400:00000200:3.0:1784949031.541116:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541122:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949031.541125:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784949031.541132:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541136:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000010:1.0:1784949031.541138:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dbce179e. 00000400:00000200:3.0:1784949031.541139:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000010:1.0:1784949031.541147:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000400:00000200:3.0:1784949031.541148:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949031.541155:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000383d6bd1. 00000400:00000200:3.0:1784949031.541159:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:3.0:1784949031.541162:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949031.541167:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784949031.541168:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541173:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541175:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784949031.541177:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:1.0:1784949031.541183:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:3.0:1784949031.541183:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541192:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.541195:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.541197:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.541200:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:1.0:1784949031.541202:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3528:3528:251:4294967295] 111.222.173.2@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:1.0:1784949031.541205:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:1.0:1784949031.541209:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.541213:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784949031.541217:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000800:00000010:1.0:1784949031.541218:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000400:00000200:3.0:1784949031.541223:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541227:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949031.541229:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949031.541236:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541243:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000800:00000200:1.1:1784949031.541243:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949031.541247:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:3.0:1784949031.541251:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949031.541253:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:3.0:1784949031.541255:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541256:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541258:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541259:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:3.0:1784949031.541261:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:3.0:1784949031.541266:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541274:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:3.0:1784949031.541275:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784949031.541280:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949031.541284:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:3.0:1784949031.541284:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541292:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541295:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:3.0:1784949031.541297:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:1.0:1784949031.541301:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784949031.541306:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949031.541309:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf7eadde. 00000400:00000200:3.0:1784949031.541313:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:1.0:1784949031.541315:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007d821c01. 00000400:00000200:3.0:1784949031.541316:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541323:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541325:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541327:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000010:1.0:1784949031.541328:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003d75c981. 00000400:00000200:3.0:1784949031.541329:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:3.0:1784949031.541341:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541343:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784949031.541348:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000400:00000200:3.0:1784949031.541350:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541353:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541356:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541358:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:3.0:1784949031.541359:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:1.0:1784949031.541363:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:3.0:1784949031.541365:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541371:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.541375:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.541377:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.541379:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:1.0:1784949031.541382:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3529:3529:250:4294967295] 111.222.173.3@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:1.0:1784949031.541385:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:1.0:1784949031.541389:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.541394:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784949031.541397:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000800:00000010:1.0:1784949031.541398:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000400:00000200:3.0:1784949031.541400:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541405:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949031.541410:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949031.541411:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541419:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784949031.541423:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000800:00000200:1.1:1784949031.541424:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949031.541429:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949031.541434:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:3.0:1784949031.541435:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541437:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541439:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541441:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:1.0:1784949031.541445:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784949031.541447:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:1.0:1784949031.541452:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:3.0:1784949031.541457:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541462:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:3.0:1784949031.541465:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:3.0:1784949031.541468:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541471:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:1.0:1784949031.541473:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f7a7941e. 00000400:00000200:3.0:1784949031.541477:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000010:1.0:1784949031.541480:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000400:00000200:3.0:1784949031.541482:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784949031.541487:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000010:1.0:1784949031.541489:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a7ce911a. 00000400:00000200:3.0:1784949031.541492:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541498:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784949031.541508:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:3.0:1784949031.541511:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541515:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541521:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541523:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:3.0:1784949031.541525:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:1.0:1784949031.541529:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:3.0:1784949031.541532:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541537:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.541540:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.541543:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.541545:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:1.0:1784949031.541549:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3530:3530:249:4294967295] 111.222.142.3@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:1.0:1784949031.541554:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:1.0:1784949031.541564:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.541568:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:1.0:1784949031.541571:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000400:00000200:3.0:1784949031.541581:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000800:00000200:1.0:1784949031.541584:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949031.541586:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.0:1784949031.541591:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 9 secs later) 00000400:00000200:3.0:1784949031.541592:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541602:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000800:00000200:1.1:1784949031.541603:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000800:00000200:1.1:1784949031.541607:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 9 secs later) 00000800:00000100:1.0:1784949031.541611:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.3@tcp 00000800:00000010:1.0:1784949031.541613:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:1.0:1784949031.541616:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.541619:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949031.541639:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:1.0:1784949031.541641:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949031.541643:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:1.0:1784949031.541648:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:3.0:1784949031.541653:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541658:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000200:1.0:1784949031.541667:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:1.0:1784949031.541671:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 rc 0 00000400:00000200:3.0:1784949031.541674:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784949031.541678:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541681:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949031.541685:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784949031.541704:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541710:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000010:1.0:1784949031.541713:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000378c7327. 00000400:00000200:3.0:1784949031.541714:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000010:1.0:1784949031.541721:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000400:00000200:3.0:1784949031.541722:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949031.541732:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784949031.541734:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949031.541737:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003c8b2437. 00000400:00000200:3.0:1784949031.541737:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541744:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:1.0:1784949031.541750:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784949031.541752:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:3.0:1784949031.541756:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:1.0:1784949031.541760:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:3.0:1784949031.541763:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541769:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.541773:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.541775:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.541777:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:1.0:1784949031.541780:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3531:3531:249:4294967295] 111.222.173.4@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:1.0:1784949031.541783:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:1.0:1784949031.541787:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.541791:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000800:00000010:1.0:1784949031.541796:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000400:00000200:3.0:1784949031.541796:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:3.0:1784949031.541800:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541804:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949031.541806:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784949031.541812:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000800:00000200:1.1:1784949031.541818:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784949031.541819:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:3.0:1784949031.541827:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:1.0:1784949031.541828:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:1.0:1784949031.541834:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784949031.541837:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541845:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000200:3.0:1784949031.541847:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:3.0:1784949031.541852:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541855:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:1.0:1784949031.541856:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf393d23. 00000400:00000200:3.0:1784949031.541859:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000010:1.0:1784949031.541861:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000200:3.0:1784949031.541863:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000200:3.0:1784949031.541868:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000010:1.0:1784949031.541871:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000004d10c17. 00000400:00000200:3.0:1784949031.541873:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541879:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:3.0:1784949031.541881:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784949031.541885:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.541888:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541891:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.541893:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:3.0:1784949031.541895:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:1.0:1784949031.541899:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:3.0:1784949031.541929:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.541930:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.541938:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.541941:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.541944:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:1.0:1784949031.541947:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3532:3532:248:4294967295] 111.222.142.4@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:1.0:1784949031.541951:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:1.0:1784949031.541955:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.541959:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:1.0:1784949031.541962:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000400:00000200:3.0:1784949031.541964:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:3.0:1784949031.541968:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.0:1784949031.541973:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:3.0:1784949031.541973:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.541985:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000800:00000200:1.1:1784949031.541985:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:3.0:1784949031.541993:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000200:1.0:1784949031.541996:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:3.0:1784949031.541999:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:1.0:1784949031.542004:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x261 rc 0 00000400:00000200:3.0:1784949031.542009:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.542014:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000200:3.0:1784949031.542018:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:3.0:1784949031.542021:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.542023:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:1.0:1784949031.542024:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000072ce5ef0. 00000400:00000200:3.0:1784949031.542028:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000010:1.0:1784949031.542031:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000400:00000200:3.0:1784949031.542033:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:3.0:1784949031.542037:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000010:1.0:1784949031.542039:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000096383a5. 00000400:00000200:3.0:1784949031.542040:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.542051:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:3.0:1784949031.542052:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784949031.542057:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.542061:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949031.542063:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.542065:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784949031.542066:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000200:1.0:1784949031.542070:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:3.0:1784949031.542072:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949031.542077:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542081:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542083:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542085:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:1.0:1784949031.542087:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3533:3533:247:4294967295] 111.222.173.5@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:1.0:1784949031.542090:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:1.0:1784949031.542094:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542099:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000400:00000200:3.0:1784949031.542103:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000800:00000010:1.0:1784949031.542105:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000400:00000200:3.0:1784949031.542106:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949031.542112:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949031.542116:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:3.0:1784949031.542120:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949031.542124:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000800:00000200:1.1:1784949031.542125:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000100:00000010:3.0:1784949031.542128:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 000000008142e1f1. 00000020:00000001:3.0:1784949031.542137:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00010000:00000001:3.0:1784949031.542140:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000100:00000001:3.0:1784949031.542143:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000010:3.0:1784949031.542150:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 0000000015713dee. 00000400:00000200:1.0:1784949031.542151:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:1.0:1784949031.542156:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x261 rc 0 00000400:00000200:3.0:1784949031.542161:0:343529:0:(peer.c:266:lnet_peer_alloc()) 0000000015713dee nid 111.222.173.28@tcp 00000400:00000200:1.0:1784949031.542163:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000010:3.0:1784949031.542167:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000810fb56e. 00000400:00000001:1.0:1784949031.542168:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.542173:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949031.542173:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 00000000810fb56e net tcp 00000400:00000200:1.0:1784949031.542176:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:3.0:1784949031.542179:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 0000000070ada26c. 00000400:00000010:1.0:1784949031.542180:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000400:00000200:1.0:1784949031.542188:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949031.542190:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 0000000070ada26c nid 111.222.173.28@tcp 00000400:00000200:1.0:1784949031.542195:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:1.0:1784949031.542199:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000035b54a67. 00000400:00000200:1.0:1784949031.542206:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-5 00000400:00000200:1.0:1784949031.542209:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -5 00000400:00000200:3.0:1784949031.542216:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.173.28@tcp NID 111.222.173.28@tcp flags 0x0 00000400:00000200:3.0:1784949031.542221:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.173.28@tcp: 0 00000400:00000200:1.0:1784949031.542224:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:1.0:1784949031.542233:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000200:3.0:1784949031.542239:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.28@tcp: 0 00000400:00000200:3.0:1784949031.542242:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949031.542247:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000035b54a67. 00000400:00000010:1.0:1784949031.542252:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000400:00000010:1.0:1784949031.542258:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000756d087e. 00000400:00000200:1.0:1784949031.542265:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:1.0:1784949031.542274:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:1.0:1784949031.542279:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542282:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542285:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542288:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:1.0:1784949031.542292:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3534:3534:246:4294967295] 111.222.142.5@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:1.0:1784949031.542297:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:1.0:1784949031.542308:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542313:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:1.0:1784949031.542318:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000800:00000200:1.0:1784949031.542323:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.0:1784949031.542326:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 9 secs later) 00000800:00000200:1.1:1784949031.542331:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:1.1:1784949031.542333:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 9 secs later) 00000800:00000100:1.0:1784949031.542338:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.5@tcp 00000800:00000010:1.0:1784949031.542340:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:1.0:1784949031.542344:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.542347:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949031.542349:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949031.542351:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:1.0:1784949031.542354:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:1.0:1784949031.542359:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:1.0:1784949031.542363:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 rc 0 00000400:00000200:1.0:1784949031.542367:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:1.0:1784949031.542371:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000052e1e01a. 00000400:00000010:1.0:1784949031.542374:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000010:1.0:1784949031.542377:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000dceddabe. 00000400:00000200:1.0:1784949031.542381:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:1.0:1784949031.542385:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:1.0:1784949031.542387:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542389:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542390:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542392:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:1.0:1784949031.542395:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3535:3535:246:4294967295] 111.222.173.6@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:1.0:1784949031.542398:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:1.0:1784949031.542402:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542405:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:1.0:1784949031.542407:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000800:00000200:1.0:1784949031.542410:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784949031.542412:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:1.0:1784949031.542415:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:1.0:1784949031.542416:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:1.0:1784949031.542418:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:1.0:1784949031.542420:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dd0652fb. 00000400:00000010:1.0:1784949031.542422:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000400:00000010:1.0:1784949031.542424:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000067f2d25d. 00000400:00000200:1.0:1784949031.542426:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:1.0:1784949031.542429:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:1.0:1784949031.542431:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542433:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542435:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542437:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:1.0:1784949031.542439:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3536:3536:245:4294967295] 111.222.173.7@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:1.0:1784949031.542443:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:1.0:1784949031.542447:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542450:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:1.0:1784949031.542454:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ca1124f5. 00000800:00000200:1.0:1784949031.542457:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784949031.542459:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:1.0:1784949031.542471:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:1.0:1784949031.542473:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000400:00000200:1.0:1784949031.542475:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:1.0:1784949031.542477:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef171f31. 00000400:00000010:1.0:1784949031.542479:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000010:1.0:1784949031.542483:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000147b89d3. 00000400:00000200:1.0:1784949031.542486:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:1.0:1784949031.542490:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:1.0:1784949031.542494:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542496:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542498:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542502:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:1.0:1784949031.542505:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3537:3537:244:4294967295] 111.222.142.7@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:1.0:1784949031.542508:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:1.0:1784949031.542513:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542518:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:1.0:1784949031.542522:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b9b32818. 00000800:00000200:1.0:1784949031.542527:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.0:1784949031.542529:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 9 secs later) 00000800:00000200:1.1:1784949031.542532:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784949031.542534:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 9 secs later) 00000800:00000100:1.0:1784949031.542538:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.7@tcp 00000800:00000010:1.0:1784949031.542540:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b9b32818. 00000400:00000200:1.0:1784949031.542541:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.542544:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949031.542546:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949031.542548:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:1.0:1784949031.542552:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:1.0:1784949031.542558:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:1.0:1784949031.542560:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 rc 0 00000400:00000200:1.0:1784949031.542562:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:1.0:1784949031.542571:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f0621080. 00000400:00000010:1.0:1784949031.542574:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000400:00000010:1.0:1784949031.542578:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000038aa1d2e. 00000400:00000200:1.0:1784949031.542581:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:1.0:1784949031.542585:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:1.0:1784949031.542587:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542589:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542591:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542593:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:1.0:1784949031.542595:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3538:3538:244:4294967295] 111.222.142.8@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:1.0:1784949031.542598:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:1.0:1784949031.542602:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542605:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:1.0:1784949031.542606:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b9b32818. 00000800:00000200:1.0:1784949031.542609:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784949031.542611:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000400:00000200:1.0:1784949031.542614:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:1.0:1784949031.542615:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x261 rc 0 00000400:00000200:1.0:1784949031.542617:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:1.0:1784949031.542620:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000032fe57fd. 00000400:00000010:1.0:1784949031.542622:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 00000400:00000010:1.0:1784949031.542624:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009e875900. 00000400:00000200:1.0:1784949031.542626:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:1.0:1784949031.542629:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:1.0:1784949031.542630:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542632:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542634:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542636:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:1.0:1784949031.542638:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3539:3539:243:4294967295] 111.222.173.9@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:1.0:1784949031.542643:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:1.0:1784949031.542647:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542650:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:1.0:1784949031.542653:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cf155122. 00000800:00000200:1.0:1784949031.542656:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784949031.542658:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000400:00000200:1.0:1784949031.542671:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:1.0:1784949031.542673:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x261 rc 0 00000400:00000200:1.0:1784949031.542676:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:1.0:1784949031.542678:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:1.0:1784949031.542680:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000400:00000010:1.0:1784949031.542684:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005547c433. 00000400:00000200:1.0:1784949031.542688:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:1.0:1784949031.542711:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:1.0:1784949031.542716:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542719:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542720:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542723:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:1.0:1784949031.542726:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3540:3540:242:4294967295] 111.222.142.9@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:1.0:1784949031.542730:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:1.0:1784949031.542742:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542749:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:1.0:1784949031.542755:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f495c80e. 00000800:00000200:1.0:1784949031.542761:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.0:1784949031.542765:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 9 secs later) 00000800:00000200:1.1:1784949031.542770:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784949031.542773:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 9 secs later) 00000800:00000100:1.0:1784949031.542779:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.9@tcp 00000800:00000010:1.0:1784949031.542782:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f495c80e. 00000400:00000200:1.0:1784949031.542787:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.542792:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949031.542796:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949031.542799:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:1.0:1784949031.542803:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000200:1.0:1784949031.542810:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:1.0:1784949031.542813:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 rc 0 00000400:00000200:1.0:1784949031.542817:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:1.0:1784949031.542823:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000467d4d37. 00000400:00000010:1.0:1784949031.542827:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000400:00000010:1.0:1784949031.542831:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000735cb99e. 00000400:00000200:1.0:1784949031.542835:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:1.0:1784949031.542842:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:1.0:1784949031.542846:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542849:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542852:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542856:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:1.0:1784949031.542859:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3541:3541:242:4294967295] 111.222.173.10@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:1.0:1784949031.542865:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:1.0:1784949031.542874:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542880:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:1.0:1784949031.542883:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f495c80e. 00000800:00000200:1.0:1784949031.542886:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784949031.542890:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:1.0:1784949031.542893:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:1.0:1784949031.542894:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:1.0:1784949031.542897:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:1.0:1784949031.542900:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000bbdbdf4. 00000400:00000010:1.0:1784949031.542902:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000010:1.0:1784949031.542940:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c3236cd0. 00000400:00000200:1.0:1784949031.542943:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:1.0:1784949031.542949:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:1.0:1784949031.542953:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.542957:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.542960:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.542963:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:1.0:1784949031.542966:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3542:3542:241:4294967295] 111.222.142.10@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:1.0:1784949031.542970:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:1.0:1784949031.542974:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.542977:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:1.0:1784949031.542979:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007a3f8609. 00000800:00000200:1.0:1784949031.542982:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784949031.542986:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000400:00000200:1.0:1784949031.542988:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:1.0:1784949031.542990:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x261 rc 0 00000400:00000200:1.0:1784949031.542992:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:1.0:1784949031.542994:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a5bcc72e. 00000400:00000010:1.0:1784949031.542996:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000400:00000010:1.0:1784949031.542999:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006b55cc75. 00000400:00000200:1.0:1784949031.543001:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:1.0:1784949031.543004:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:1.0:1784949031.543006:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543008:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543010:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543012:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:1.0:1784949031.543014:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3543:3543:240:4294967295] 111.222.173.11@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:1.0:1784949031.543017:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:1.0:1784949031.543021:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543023:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:1.0:1784949031.543025:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a21d8ef3. 00000800:00000200:1.0:1784949031.543028:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784949031.543031:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:1.0:1784949031.543043:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:1.0:1784949031.543045:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:1.0:1784949031.543047:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 00000400:00000001:1.0:1784949031.543049:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.543050:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.543051:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000010:1.0:1784949031.543052:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000400:00000200:1.0:1784949031.543054:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.543058:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:1.0:1784949031.543060:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f7a7941e. 00000400:00000200:1.0:1784949031.543064:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-5 00000400:00000200:1.0:1784949031.543065:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -5 00000400:00000200:1.0:1784949031.543068:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00000400:00000200:1.0:1784949031.543071:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:1.0:1784949031.543074:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f7a7941e. 00000400:00000010:1.0:1784949031.543077:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000400:00000010:1.0:1784949031.543080:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000063d06960. 00000400:00000200:1.0:1784949031.543084:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:1.0:1784949031.543089:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:1.0:1784949031.543092:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543095:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543099:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543102:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:1.0:1784949031.543105:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3544:3544:239:4294967295] 111.222.142.11@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:1.0:1784949031.543109:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:1.0:1784949031.543125:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543128:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:1.0:1784949031.543131:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:1.0:1784949031.543135:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.0:1784949031.543139:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 9 secs later) 00000800:00000200:1.1:1784949031.543155:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784949031.543158:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 9 secs later) 00000800:00000100:1.0:1784949031.543161:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.11@tcp 00000800:00000010:1.0:1784949031.543163:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:1.0:1784949031.543166:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.543172:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949031.543175:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949031.543177:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:1.0:1784949031.543179:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:1.0:1784949031.543188:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:1.0:1784949031.543190:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 rc 0 00000400:00000200:1.0:1784949031.543197:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:1.0:1784949031.543201:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007db69e48. 00000400:00000010:1.0:1784949031.543204:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000400:00000010:1.0:1784949031.543207:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000a5d96d2. 00000400:00000200:1.0:1784949031.543209:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:1.0:1784949031.543212:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:1.0:1784949031.543214:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543216:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543218:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543219:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:1.0:1784949031.543221:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3545:3545:239:4294967295] 111.222.173.12@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:1.0:1784949031.543224:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:1.0:1784949031.543228:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543230:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:1.0:1784949031.543232:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:1.0:1784949031.543235:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784949031.543238:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:1.0:1784949031.543241:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:1.0:1784949031.543242:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:1.0:1784949031.543245:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:1.0:1784949031.543249:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001a4eba99. 00000400:00000010:1.0:1784949031.543251:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000010:1.0:1784949031.543254:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000056670828. 00000400:00000200:1.0:1784949031.543256:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:1.0:1784949031.543259:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:1.0:1784949031.543261:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543262:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543264:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543265:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:1.0:1784949031.543267:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3546:3546:238:4294967295] 111.222.142.12@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:1.0:1784949031.543270:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:1.0:1784949031.543274:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543278:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:1.0:1784949031.543280:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000800:00000200:1.0:1784949031.543283:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784949031.543286:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:1.0:1784949031.543289:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:1.0:1784949031.543290:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:1.0:1784949031.543292:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:1.0:1784949031.543294:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000072ce2c8. 00000400:00000010:1.0:1784949031.543296:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000400:00000010:1.0:1784949031.543299:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001fcbecf4. 00000400:00000200:1.0:1784949031.543301:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:1.0:1784949031.543303:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:1.0:1784949031.543305:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543306:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543308:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543309:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:1.0:1784949031.543311:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3547:3547:237:4294967295] 111.222.173.13@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:1.0:1784949031.543314:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:1.0:1784949031.543318:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543322:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:1.0:1784949031.543324:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cee99234. 00000800:00000200:1.0:1784949031.543326:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784949031.543330:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:1.0:1784949031.543340:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:1.0:1784949031.543342:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:1.0:1784949031.543345:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:1.0:1784949031.543348:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000063f28b6f. 00000400:00000010:1.0:1784949031.543350:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000010:1.0:1784949031.543354:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000da72cb2d. 00000400:00000200:1.0:1784949031.543357:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:1.0:1784949031.543361:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:1.0:1784949031.543364:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543366:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543370:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543373:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:1.0:1784949031.543375:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3548:3548:236:4294967295] 111.222.142.13@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:1.0:1784949031.543379:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:1.0:1784949031.543384:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543387:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:1.0:1784949031.543391:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000800:00000200:1.0:1784949031.543395:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.0:1784949031.543396:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.13 (cnted 0, interval 60, 26 secs later) 00000800:00000200:1.1:1784949031.543401:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784949031.543403:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.13 (cnted 0, interval 60, 26 secs later) 00000800:00000100:1.0:1784949031.543414:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.13@tcp 00000800:00000010:1.0:1784949031.543415:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000200:1.0:1784949031.543418:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.543421:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949031.543425:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949031.543427:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:1.0:1784949031.543430:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:1.0:1784949031.543435:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:1.0:1784949031.543436:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 rc 0 00000400:00000200:1.0:1784949031.543440:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:1.0:1784949031.543446:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000059e5b358. 00000400:00000010:1.0:1784949031.543449:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000400:00000010:1.0:1784949031.543453:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000038468d93. 00000400:00000200:1.0:1784949031.543456:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:1.0:1784949031.543460:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:1.0:1784949031.543462:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543465:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543467:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543468:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:1.0:1784949031.543471:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3549:3549:236:4294967295] 111.222.173.14@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:1.0:1784949031.543474:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:1.0:1784949031.543478:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543482:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:1.0:1784949031.543484:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000800:00000200:1.0:1784949031.543487:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:1.1:1784949031.543490:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:1.0:1784949031.543493:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:1.0:1784949031.543494:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:1.0:1784949031.543497:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:1.0:1784949031.543501:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000010:1.0:1784949031.543502:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000010:1.0:1784949031.543505:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008c245eeb. 00000400:00000200:1.0:1784949031.543508:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:1.0:1784949031.543510:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:1.0:1784949031.543512:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543513:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543515:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543517:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:1.0:1784949031.543519:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3550:3550:235:4294967295] 111.222.142.14@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:1.0:1784949031.543522:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:1.0:1784949031.543527:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543530:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:1.0:1784949031.543532:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000800:00000200:1.0:1784949031.543535:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784949031.543537:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:1.0:1784949031.543540:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:1.0:1784949031.543541:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:1.0:1784949031.543543:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:1.0:1784949031.543546:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009802f36f. 00000400:00000010:1.0:1784949031.543548:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000010:1.0:1784949031.543550:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000acbf6818. 00000400:00000200:1.0:1784949031.543553:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:1.0:1784949031.543555:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:1.0:1784949031.543557:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543559:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543561:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543563:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:1.0:1784949031.543565:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3551:3551:234:4294967295] 111.222.173.15@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:1.0:1784949031.543568:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:1.0:1784949031.543572:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543574:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:1.0:1784949031.543576:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000005678171. 00000800:00000200:1.0:1784949031.543579:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784949031.543582:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:1.0:1784949031.543596:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:1.0:1784949031.543598:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:1.0:1784949031.543600:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:1.0:1784949031.543603:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf0e6e18. 00000400:00000010:1.0:1784949031.543604:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000010:1.0:1784949031.543607:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000451e3fe4. 00000400:00000200:1.0:1784949031.543609:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:1.0:1784949031.543612:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:1.0:1784949031.543613:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949031.543615:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543618:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949031.543621:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:1.0:1784949031.543623:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3552:3552:233:4294967295] 111.222.142.15@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:1.0:1784949031.543627:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:1.0:1784949031.543633:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543637:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:1.0:1784949031.543640:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004b264730. 00000800:00000200:1.0:1784949031.543648:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.0:1784949031.543650:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.15 (cnted 0, interval 60, 26 secs later) 00000800:00000200:1.1:1784949031.543654:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784949031.543656:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.15 (cnted 0, interval 60, 26 secs later) 00000800:00000100:1.0:1784949031.543659:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.15@tcp 00000800:00000010:1.0:1784949031.543660:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004b264730. 00000400:00000200:1.0:1784949031.543662:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949031.543665:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949031.543669:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949031.543671:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:1.0:1784949031.543677:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:1.0:1784949031.543685:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:1.0:1784949031.543687:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 rc 0 00000400:00000200:1.0:1784949031.543689:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.28@tcp(0000000015713dee) state 0x40 00000400:00000010:1.0:1784949031.543712:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004f865a97. 00000400:00000010:1.0:1784949031.543719:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000010:1.0:1784949031.543722:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009234a5fd. 00000400:00000200:1.0:1784949031.543725:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.28@tcp 00000400:00000200:1.0:1784949031.543732:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.173.28@tcp local destination 00000400:00000200:1.0:1784949031.543735:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949031.543737:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.173.28@tcp 00000400:00000200:1.0:1784949031.543741:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.173.28@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:1.0:1784949031.543744:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3553:3553:233:4294967295] 111.222.173.28@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:1.0:1784949031.543747:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.28@tcp 00000400:00000200:1.0:1784949031.543752:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.28@tcp(111.222.173.28@tcp:111.222.173.28@tcp) : GET try# 0 00000800:00000200:1.0:1784949031.543757:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.28@tcp 00000800:00000010:1.0:1784949031.543760:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004b264730. 00000800:00000010:1.0:1784949031.543771:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000e4acfb45. 00000800:00000010:1.0:1784949031.543776:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000c5dc258d. 00000800:00000200:1.0:1784949031.543783:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000e4acfb45] -> 12345-111.222.173.28@tcp (2) 00000800:00000200:1.1:1784949031.543788:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000e4acfb45] -> 12345-111.222.173.28@tcp (2) 00000400:00000200:1.0:1784949031.543805:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.28@tcp 00000400:00000200:1.0:1784949031.543808:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.28@tcp(0000000015713dee) state 0x240 rc 0 00000400:00000200:1.0:1784949031.543813:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:1.0:1784949031.543817:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.543820:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.543822:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:1.0:1784949031.543824:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000400:00000200:1.0:1784949031.543827:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.543830:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:1.0:1784949031.543833:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000035b54a67. 00000400:00000200:1.0:1784949031.543837:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-5 00000400:00000200:1.0:1784949031.543839:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -5 00000400:00000200:1.0:1784949031.543842:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000400:00000200:1.0:1784949031.543845:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000400:00000001:1.0:1784949031.543848:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.543849:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.543850:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:1.0:1784949031.543852:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000400:00000200:1.0:1784949031.543853:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.543858:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:1.0:1784949031.543861:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef171f31. 00000400:00000200:1.0:1784949031.543864:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-5 00000400:00000200:1.0:1784949031.543865:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -5 00000400:00000200:1.0:1784949031.543867:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:1.0:1784949031.543870:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:1.0:1784949031.543872:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.543873:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.543874:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000400:00000010:1.0:1784949031.543876:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000400:00000200:1.0:1784949031.543880:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.543882:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:1.0:1784949031.543884:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000400:00000200:1.0:1784949031.543887:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-5 00000400:00000200:1.0:1784949031.543889:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -5 00000400:00000200:1.0:1784949031.543892:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:1.0:1784949031.543894:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:1.0:1784949031.543895:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.543896:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.543897:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000010:1.0:1784949031.543898:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000400:00000200:1.0:1784949031.543900:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.543902:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:1.0:1784949031.543933:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f7a7941e. 00000400:00000200:1.0:1784949031.543938:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-5 00000400:00000200:1.0:1784949031.543941:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -5 00000400:00000200:1.0:1784949031.543945:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000400:00000200:1.0:1784949031.543949:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:1.0:1784949031.543951:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.543953:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.543954:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000010:1.0:1784949031.543956:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000400:00000200:1.0:1784949031.543958:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.543960:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:1.0:1784949031.543961:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000063f28b6f. 00000400:00000200:1.0:1784949031.543964:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-5 00000400:00000200:1.0:1784949031.543965:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -5 00000400:00000200:1.0:1784949031.543967:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:1.0:1784949031.543969:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:1.0:1784949031.543971:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949031.543982:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949031.543983:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:1.0:1784949031.543984:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000400:00000200:1.0:1784949031.543986:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949031.543990:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:1.0:1784949031.543992:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf0e6e18. 00000400:00000200:1.0:1784949031.543996:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-5 00000400:00000200:1.0:1784949031.543997:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -5 00000400:00000200:1.0:1784949031.544001:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784949032.180097:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949032.180107:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949032.180116:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949032.180118:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949032.180222:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949032.180229:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949032.180233:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:3.0:1784949032.564220:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949032.564230:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000f682530a. 00000800:00000100:2.0:1784949032.564243:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.2@tcp 00000800:00000010:2.0:1784949032.564250:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000076916763. 00000400:00000010:3.0:1784949032.564251:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000200:2.0:1784949032.564260:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949032.564261:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000400:00000200:2.0:1784949032.564277:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784949032.564282:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c986c2b1. 00000400:00000200:2.0:1784949032.564287:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11305 health :700 00000400:00000200:3.0:1784949032.564298:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949032.564305:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949032.564310:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:3.0:1784949032.564321:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949032.564329:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949032.564334:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3554:3554:232:4294967295] 192.168.202.137@tcp LPNI seq info [2279:2279:8:4294967295] 00000400:00000200:3.0:1784949032.564490:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949032.564511:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949032.564519:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949032.564523:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000800:00000010:3.0:1784949032.564530:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949032.564543:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949032.564558:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949032.564574:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000800:00000100:2.0:1784949032.564609:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000800:00000010:2.0:1784949032.564616:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c53910d9. 00000400:00000200:2.0:1784949032.564629:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949032.564640:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949032.564653:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949032.564660:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:1.0:1784949032.564661:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949032.564670:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:1.0:1784949032.564705:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2861 00000400:00000001:1.0:1784949032.564716:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949032.564720:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949032.564724:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:1.0:1784949032.564739:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000400:00000200:1.0:1784949032.564746:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949032.564752:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:1.0:1784949032.564758:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ead2c724. 00000400:00000200:1.0:1784949032.564764:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-110 00000400:00000200:1.0:1784949032.564767:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -110 00000400:00000200:1.0:1784949032.564772:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:1.0:1784949032.564778:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2861 00000400:00000001:1.0:1784949032.564781:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949032.564783:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949032.564785:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:1.0:1784949032.564787:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:1.0:1784949032.564790:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949032.564793:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:1.0:1784949032.564795:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000053ab16eb. 00000400:00000200:1.0:1784949032.564798:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000200:1.0:1784949032.564800:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:1.0:1784949032.564803:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000800:00000001:3.0:1784949032.564836:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949032.564838:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949032.564846:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:0.0:1784949032.564854:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949032.564854:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949032.564860:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949032.564866:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:3.0:1784949032.564873:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949032.564874:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949032.564883:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:3.0:1784949032.564889:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949032.564933:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949032.564945:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949032.564956:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949032.564961:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3555:3555:234:4294967295] 192.168.202.137@tcp LPNI seq info [2280:2280:8:4294967295] 00000400:00000200:3.0:1784949032.564970:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949032.564975:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949032.564979:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784949032.564984:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949032.564989:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949032.564992:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 3, nob 144 niov 1 nkiov 1 00000800:00000200:3.0:1784949032.565065:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949032.565068:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:3.0:1784949032.565074:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949032.565079:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949032.565096:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949032.565096:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949032.565110:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949032.565111:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949032.565121:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949032.565132:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949032.565141:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949032.565158:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8c45 00000800:00000001:3.0:1784949032.565167:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949032.565175:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949032.565178:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949032.565184:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949032.565190:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:3.0:1784949032.565192:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000400:00000200:3.0:1784949032.565196:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949032.565199:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000f682530a. 00000400:00000010:3.0:1784949032.565202:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000800:00000001:3.0:1784949032.565209:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949032.565212:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949033.204025:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949033.204034:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949033.204042:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949033.204043:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949033.204150:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949033.204159:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949033.204163:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1784949033.588067:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.6@tcp 00000800:00000010:2.0:1784949033.588073:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000094ed938f. 00000400:00000200:2.0:1784949033.588080:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949033.588085:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949033.588093:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11306 health :900 00000400:00000200:2.0:1784949033.588097:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949033.588100:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:2.0:1784949033.588104:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:1.0:1784949033.588165:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949033.588181:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2861 00000400:00000001:1.0:1784949033.588191:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949033.588194:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949033.588198:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:1.0:1784949033.588201:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000400:00000200:1.0:1784949033.588209:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949033.588214:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:1.0:1784949033.588219:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000011003cc7. 00000400:00000200:1.0:1784949033.588226:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-110 00000400:00000200:1.0:1784949033.588229:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -110 00000400:00000200:1.0:1784949033.588233:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00010000:00000001:2.0:1784949034.228055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949034.228066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949034.228076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949034.228078:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949034.228180:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949034.228190:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949034.228197:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949034.420023:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949034.420035:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949034.420047:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000617daa1a. 00000100:00000001:2.0:1784949034.420050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000020:00000040:0.0:1784949034.420055:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949034.420061:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949034.420065:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949034.420066:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949034.420073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949034.420076:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.420079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949034.420083:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949034.420087:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949034.420088:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000086e4d21. 00000100:00000001:2.0:1784949034.420124:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.420130:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949034.420144:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949034.420159:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949034.420162:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949034.420170:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000617daa1a] to pc [ptlrpcd_00_03+3] req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949034.420197:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949034.420199:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949034.420203:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949034.420207:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949034.420211:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949034.420214:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784949034.420215:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000020:00000040:0.0:1784949034.420220:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949034.420222:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949034.420223:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949034.420227:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949034.420227:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784949034.420230:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949034.420235:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949034.420236:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b84187a1. 00000100:00000001:0.0:1784949034.420241:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.420250:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949034.420251:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784949034.420252:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949034.420262:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.420266:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949034.420270:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949034.420274:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949034.420279:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949034.420281:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949034.420285:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949034.420289:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_00_00+0] req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949034.420291:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289669760:0@lo:400: 00000100:00000001:1.0:1784949034.420304:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949034.420314:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949034.420317:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949034.420320:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949034.420327:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 02000000:00000001:1.0:1784949034.420329:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.420332:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949034.420337:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e572277. 00000100:00000040:0.0:1784949034.420339:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11311) 00000400:00000010:1.0:1784949034.420349:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000001:2.0:1784949034.420353:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784949034.420359:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289669760, portal 25 00000100:00000001:2.0:1784949034.420360:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949034.420367:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949034.420368:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949034.420390:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554966272 : -115544154585344 : ffff96e9c7b5c700) 00000100:00000001:0.0:1784949034.420396:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949034.420401:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949034.420402:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949034.420403:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.420468:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949034.420484:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784949034.420485:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949034.420494:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949034.420497:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000100:00000200:1.0:1784949034.420505:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289669760, offset 0 02000000:00000001:2.0:1784949034.420506:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949034.420517:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949034.420529:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784949034.420531:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949034.420542:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289669888:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949034.420549:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:1.0:1784949034.420557:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784949034.420560:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949034.420564:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949034.420570:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949034.420576:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a080 02000000:00000010:2.0:1784949034.420581:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 02000000:00000001:2.0:1784949034.420587:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949034.420598:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8229 [2] + 0 00000400:00000010:1.0:1784949034.420605:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000aee1f7f7. 00000400:00000010:2.0:1784949034.420607:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dee4e850. 00000400:00000200:1.0:1784949034.420612:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000553d6b82 00000400:00000010:2.0:1784949034.420623:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000400:00000200:1.0:1784949034.420626:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949034.420636:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:2.0:1784949034.420637:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289669888, portal 10 00000100:00000001:2.0:1784949034.420651:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949034.420657:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553d6b82 00000100:00000001:2.0:1784949034.420657:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 00000400:00000010:1.0:1784949034.420689:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000553d6b82. 00000100:00000001:1.0:1784949034.420724:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949034.420728:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949034.420728:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949034.420743:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000322d6a4a x1871651289669760 msgsize 224 00000100:00100000:1.0:1784949034.420770:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:2.0:1784949034.420770:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949034.420778:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:2.0:1784949034.420786:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949034.420790:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000001:1.0:1784949034.420793:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:2.0:1784949034.420795:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289669888, offset 0 00000400:00000200:2.0:1784949034.420806:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949034.420814:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000010:1.0:1784949034.420817:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000100:00000001:1.0:1784949034.420824:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949034.420828:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949034.420831:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:1.0:1784949034.420834:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949034.420837:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289669760 00000400:00000200:2.0:1784949034.420838:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949034.420856:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949034.420858:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949034.420865:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1784949034.420866:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949034.420880:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949034.420891:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949034.420892:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3556:3556:235:4294967295] 192.168.202.137@tcp LPNI seq info [2281:2281:8:4294967295] 00000100:00000001:1.0:1784949034.420899:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784949034.420926:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:0.0:1784949034.420975:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949034.420982:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:2.0:1784949034.420982:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949034.420984:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.420993:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949034.421003:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000de6364cf. 02000000:00000001:0.0:1784949034.421004:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.421004:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305973803 00000100:00100000:0.0:1784949034.421022:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289669760 00000100:00000010:1.0:1784949034.421023:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000001:0.0:1784949034.421028:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:2.0:1784949034.421029:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000040:0.0:1784949034.421032:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949034.421033:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1784949034.421036:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949034.421042:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949034.421042:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305973803 00000800:00000200:2.0:1784949034.421047:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:0.0:1784949034.421051:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784949034.421057:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000001:0.0:1784949034.421059:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:2.0:1784949034.421060:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000de6364cf type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949034.421067:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949034.421076:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949034.421081:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:1.0:1784949034.421082:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305973803 00000100:00000001:0.0:1784949034.421087:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1784949034.421089:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000020:00000010:0.0:1784949034.421093:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00000001:2.0:1784949034.421096:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.421097:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:0.0:1784949034.421103:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:1.0:1784949034.421108:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305973803 00000100:00000040:2.0:1784949034.421110:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2136 usecs (3 jiffies) 00000020:00000010:0.0:1784949034.421111:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007194bf82. 00000100:00000010:1.0:1784949034.421116:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:1.0:1784949034.421129:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949034.421129:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305973764 00000020:00000010:0.0:1784949034.421132:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949034.421144:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305973803 00000100:00000001:1.0:1784949034.421146:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949034.421152:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305973764 00000100:00000001:1.0:1784949034.421153:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949034.421154:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949034.421159:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305973803 00000100:00000001:1.0:1784949034.421162:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.421163:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949034.421164:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305973764 00000100:00000001:1.0:1784949034.421171:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949034.421173:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949034.421176:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305973803 00000100:00000001:0.0:1784949034.421180:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.421181:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305973764 00000100:00000001:1.0:1784949034.421183:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949034.421189:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949034.421192:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305973803 00000100:00000001:1.0:1784949034.421195:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949034.421201:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.421201:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949034.421201:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949034.421209:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.421216:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.421216:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.421220:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949034.421232:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.421238:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949034.421241:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949034.421244:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949034.421248:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949034.421248:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1784949034.421256:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 761 00000100:00000001:1.0:1784949034.421258:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949034.421258:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:0.0:1784949034.421261:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949034.421263:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.421270:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180541520 : -115539529010096 : ffff96eadb6a7450) 00000100:00000001:1.0:1784949034.421271:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.421274:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949034.421281:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949034.421283:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000322d6a4a x1871651289669760/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:695/0 lens 224/0 e 0 to 0 dl 1784949045 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949034.421288:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.421302:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.421305:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949034.421309:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:0.0:1784949034.421313:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949034.421317:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000800:00000200:3.0:1784949034.421322:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784949034.421324:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:0.0:1784949034.421329:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000322d6a4a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289669760:12345-0@lo:400: 00000800:00000010:3.0:1784949034.421329:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000de6364cf. 00000100:00000200:0.0:1784949034.421352:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289669760 00000400:00000200:3.0:1784949034.421353:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949034.421357:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:1.0:1784949034.421358:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949034.421362:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949034.421363:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:3.0:1784949034.421365:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784949034.421368:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949034.421371:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949034.421372:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000020:00000001:0.0:1784949034.421373:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949034.421379:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000010:3.0:1784949034.421380:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000400:00000200:1.0:1784949034.421386:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:0.0:1784949034.421387:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949034.421390:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784949034.421392:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949034.421397:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:3.0:1784949034.421399:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949034.421401:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:1.0:1784949034.421403:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a100 00000020:00000001:0.0:1784949034.421405:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949034.421412:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:1.0:1784949034.421418:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 166128 00000100:00000001:0.0:1784949034.421419:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949034.421420:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949034.421426:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:1.0:1784949034.421430:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949034.421430:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949034.421431:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000060c7f82e. 02000000:00000001:0.0:1784949034.421438:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949034.421441:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949034.421444:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949034.421447:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949034.421448:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949034.421449:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949034.421451:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000040:0.0:1784949034.421458:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289669760 00000400:00000200:1.0:1784949034.421462:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000001:0.0:1784949034.421463:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949034.421475:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000322d6a4a x1871651289669760/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:695/0 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949034.421475:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949034.421480:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784949034.421488:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007bd58f9d. 00010000:00000001:0.0:1784949034.421489:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:1.0:1784949034.421493:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000007bd58f9d x1871651289669888 msgsize 224 00010000:00000001:0.0:1784949034.421495:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949034.421500:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=56 v=5 (1 1 1 1) 00000100:00100000:1.0:1784949034.421501:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949034.421508:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949034.421512:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949034.421513:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949034.421516:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784949034.421517:0:337114:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:0.0:1784949034.421518:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949034.421523:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:1.0:1784949034.421525:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949034.421526:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:1.0:1784949034.421532:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949034.421533:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000100:00000200:0.0:1784949034.421541:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289669760, offset 224 00000100:00000001:3.0:1784949034.421546:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:0.0:1784949034.421547:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:3.0:1784949034.421551:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289669888 02000000:00000001:3.0:1784949034.421554:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949034.421556:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:0.0:1784949034.421560:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949034.421560:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949034.421565:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949034.421569:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289669888 00000400:00000200:0.0:1784949034.421573:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a080 00000020:00000001:3.0:1784949034.421573:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949034.421576:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949034.421579:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:0.0:1784949034.421582:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8c49 [1] + 224 00000020:00000040:3.0:1784949034.421582:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949034.421588:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949034.421591:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784949034.421592:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949034.421595:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949034.421598:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:3.0:1784949034.421601:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:0.0:1784949034.421603:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949034.421607:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784949034.421608:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949034.421612:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000020:00000010:3.0:1784949034.421619:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000100:00000040:0.0:1784949034.421623:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949034.421626:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000608fe10d. 00000020:00000010:3.0:1784949034.421645:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949034.421653:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1784949034.421653:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949034.421658:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949034.421660:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784949034.421662:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000100:00000001:3.0:1784949034.421663:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949034.421681:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000100:00000001:0.0:1784949034.421686:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949034.421689:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949034.421690:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949034.421693:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.421710:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949034.421710:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949034.421715:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949034.421716:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949034.421716:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949034.421724:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:1.0:1784949034.421724:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1784949034.421729:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949034.421729:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949034.421733:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949034.421734:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:1.0:1784949034.421734:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e572277. 00000100:00000040:0.0:1784949034.421740:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000322d6a4a x1871651289669760/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:695/0 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949034.421742:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:1.0:1784949034.421746:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:3.0:1784949034.421748:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949034.421753:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949034.421753:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949034.421759:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000322d6a4a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289669760:12345-0@lo:400: Request processed in 449us (1009us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949034.421763:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 737 00000100:00000200:1.0:1784949034.421764:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949034.421770:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949034.421775:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549161600 : -115544160390016 : ffff96e9c75d3480) 00000100:00100000:0.0:1784949034.421777:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 761 00000100:00000040:0.0:1784949034.421785:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000040:3.0:1784949034.421787:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000007bd58f9d x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:695/0 lens 224/0 e 0 to 0 dl 1784949045 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949034.421792:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949034.421795:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949034.421802:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007194bf82. 00000020:00000010:0.0:1784949034.421808:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949034.421815:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000100:00000200:1.0:1784949034.421820:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949034.421822:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000040:0.0:1784949034.421824:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000010:3.0:1784949034.421827:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 02000000:00000001:0.0:1784949034.421832:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949034.421833:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949034.421837:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:0.0:1784949034.421838:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000060c7f82e. 00000020:00000010:3.0:1784949034.421838:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000100:00000001:1.0:1784949034.421840:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949034.421846:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949034.421847:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:1.0:1784949034.421848:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949034.421851:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949034.421852:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000007bd58f9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289669888:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949034.421853:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949034.421856:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949034.421857:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000200:3.0:1784949034.421861:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289669888 02000000:00000001:1.0:1784949034.421862:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949034.421865:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000968c2fa4. 00000020:00000001:3.0:1784949034.421866:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00001000:1.0:1784949034.421868:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=56 v=5 (5 5 5 5) 00000020:00000001:3.0:1784949034.421871:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:0.0:1784949034.421874:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009412aeee. 00000100:00001000:1.0:1784949034.421874:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=56 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949034.421875:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.421882:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784949034.421882:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949034.421885:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784949034.421886:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949034.421892:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949034.421892:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00010000:00000001:1.0:1784949034.421897:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949034.421898:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949034.421901:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949034.421902:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784949034.421907:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949034.421913:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949034.421916:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949034.421917:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949034.421930:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949034.421932:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949034.421933:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949034.421937:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949034.421939:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949034.421943:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003b957b1c. 02000000:00000001:3.0:1784949034.421947:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949034.421948:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949034.421952:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949034.421958:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949034.421963:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289669888 00000100:00100000:1.0:1784949034.421964:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289669760:0@lo:400: 00010000:00000001:3.0:1784949034.421967:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949034.421971:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:3.0:1784949034.421974:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000007bd58f9d x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:695/0 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949034.421975:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949034.421980:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000617daa1a x1871651289669760/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949034.421996:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949034.421998:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:3.0:1784949034.421999:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949034.422002:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00001000:3.0:1784949034.422003:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=51 v=5 (1 1 1 1) 02000000:00000010:1.0:1784949034.422005:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 00000100:00000001:3.0:1784949034.422008:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949034.422011:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949034.422012:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000001:1.0:1784949034.422015:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784949034.422016:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:1.0:1784949034.422019:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 02000000:00000001:3.0:1784949034.422020:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949034.422023:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949034.422026:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784949034.422027:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:1.0:1784949034.422029:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000086e4d21. 00000100:00000040:3.0:1784949034.422033:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949034.422035:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:3.0:1784949034.422037:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 02000000:00000001:1.0:1784949034.422038:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949034.422041:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000617daa1a. 00000100:00000200:3.0:1784949034.422043:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289669888, offset 224 00000400:00000200:3.0:1784949034.422049:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949034.422054:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949034.422057:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949034.422058:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949034.422061:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949034.422065:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1784949034.422067:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3557:3557:235:4294967295] 192.168.202.137@tcp LPNI seq info [2282:2282:8:4294967295] 00000100:00000001:1.0:1784949034.422068:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949034.422073:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949034.422076:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949034.422083:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949034.422090:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949034.422093:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784949034.422098:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949034.422104:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949034.422107:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949034.422120:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949034.422122:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949034.422124:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949034.422125:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949034.422127:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949034.422130:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007bd58f9d x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:695/0 lens 224/224 e 0 to 0 dl 1784949045 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949034.422138:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000007bd58f9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289669888:12345-192.168.202.137@tcp:400: Request processed in 315us (643us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949034.422145:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 737 00000100:00000040:3.0:1784949034.422147:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949034.422148:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949034.422150:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949034.422152:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000608fe10d. 00000020:00000010:3.0:1784949034.422156:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949034.422159:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000020:00000040:3.0:1784949034.422162:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949034.422163:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949034.422218:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784949034.422220:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949034.422224:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000800:00000001:3.0:1784949034.422225:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949034.422228:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949034.422228:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949034.422233:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949034.422237:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949034.422245:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000200:3.0:1784949034.422245:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:1.0:1784949034.422250:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000100:00000001:1.0:1784949034.422254:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949034.422256:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949034.422259:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a100 00000400:00000200:3.0:1784949034.422265:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8c51 [1] + 224 00000800:00000001:3.0:1784949034.422270:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949034.422280:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949034.422282:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949034.422285:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949034.422288:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949034.422290:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949034.422298:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949034.422312:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949034.422316:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949034.422318:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949034.422372:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949034.422376:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949034.422383:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949034.422385:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949034.422388:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dee4e850. 00000400:00000200:2.0:1784949034.422394:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:2.0:1784949034.422396:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000100:00000001:2.0:1784949034.422400:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949034.422407:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949034.422416:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949034.422423:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949034.422424:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949034.422426:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949034.422429:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949034.422432:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949034.422433:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949034.422436:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949034.422443:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=51 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949034.422446:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=51 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949034.422449:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949034.422451:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949034.422454:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949034.422458:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949034.422465:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.422468:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949034.422469:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949034.422471:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949034.422472:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949034.422475:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949034.422481:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949034.422483:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949034.422485:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949034.422487:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949034.422494:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289669888:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949034.422498:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949034.422500:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949034.422502:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289669888/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949034.422508:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949034.422511:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949034.422513:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 02000000:00000001:2.0:1784949034.422516:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949034.422518:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949034.422520:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949034.422522:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949034.422524:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b84187a1. 02000000:00000001:2.0:1784949034.422528:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949034.422529:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949034.422530:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 00000100:00000001:2.0:1784949034.422542:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949034.422543:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949034.422544:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949034.422548:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949034.422549:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949034.422551:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949034.422553:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949034.463981:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305973803, at 4305973804 00000100:00000010:1.0:1784949034.463996:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:3.0:1784949034.464017:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305973803, at 4305973804 00000100:00000010:3.0:1784949034.464027:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:2.0:1784949034.464039:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305973803, at 4305973804 00000100:00000040:0.0:1784949034.464043:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305973803, at 4305973804 00000100:00000010:2.0:1784949034.464059:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:0.0:1784949034.464063:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949035.252000:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949035.252008:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949035.252014:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949035.252016:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949035.252102:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949035.252110:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949035.252114:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949035.443997:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949035.444008:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949035.444010:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949035.444014:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949035.444017:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949035.444020:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949035.444023:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949036.276058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949036.276078:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949036.276085:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949036.276086:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949036.276217:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949036.276227:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949036.276233:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949036.468026:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949036.468042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949036.468045:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949036.468050:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949036.468053:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949036.468057:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949036.468060:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949037.300048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949037.300056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949037.300062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949037.300064:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949037.300227:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949037.300236:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949037.300242:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949037.492026:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949037.492040:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949037.492042:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949037.492047:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949037.492050:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949037.492054:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949037.492057:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949038.324084:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949038.324094:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949038.324102:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949038.324104:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949038.324257:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949038.324267:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949038.324274:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949038.516068:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949038.516084:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949038.516089:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949038.516097:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949038.516101:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949038.516108:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949038.516113:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949039.348025:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949039.348035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949039.348043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949039.348045:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949039.348132:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949039.348139:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949039.348142:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949039.540051:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1784949039.540060:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949039.540066:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949039.540073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949039.540074:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949039.540083:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949039.540086:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:3.0:1784949039.540088:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001445c4ff. 00000100:00000001:0.0:1784949039.540091:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.540097:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949039.540101:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949039.540108:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949039.540111:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949039.540118:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949039.540127:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:3.0:1784949039.540136:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949039.540152:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949039.540167:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949039.540169:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949039.540177:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001445c4ff] to pc [ptlrpcd_00_01+1] req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949039.540212:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949039.540215:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949039.540219:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949039.540224:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949039.540229:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949039.540233:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000040:3.0:1784949039.540238:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949039.540242:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949039.540246:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949039.540250:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.540253:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784949039.540257:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:0.0:1784949039.540259:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949039.540265:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.540266:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949039.540273:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949039.540281:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949039.540283:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949039.540285:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949039.540341:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949039.540342:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949039.540345:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.540347:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949039.540356:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1784949039.540360:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_02+2] req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949039.540363:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949039.540373:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949039.540383:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000001445c4ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289670016:0@lo:400: 00000100:00000001:0.0:1784949039.540393:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949039.540401:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949039.540404:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949039.540407:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949039.540414:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949039.540417:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949039.540419:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949039.540428:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000093aab5da. 00000100:00000040:3.0:1784949039.540430:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11316) 00000400:00000010:0.0:1784949039.540441:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000100:00000001:1.0:1784949039.540447:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949039.540453:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670016, portal 25 00000100:00000001:0.0:1784949039.540460:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949039.540462:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949039.540464:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554962432 : -115544154589184 : ffff96e9c7b5b800) 00000100:00000001:1.0:1784949039.540469:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949039.540477:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949039.540477:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949039.540485:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949039.540492:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949039.540517:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949039.540534:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784949039.540546:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:0.0:1784949039.540548:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000001:1.0:1784949039.540551:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949039.540556:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289670016, offset 0 02000000:00000001:1.0:1784949039.540561:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784949039.540570:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784949039.540584:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949039.540598:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:1.0:1784949039.540598:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949039.540617:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289670144:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949039.540651:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949039.540658:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949039.540661:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949039.540665:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949039.540675:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 00000400:00000200:0.0:1784949039.540677:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a180 02000000:00000001:1.0:1784949039.540681:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949039.540700:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e572277. 00000400:00000200:0.0:1784949039.540702:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8245 [2] + 0 00000400:00000010:0.0:1784949039.540715:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b32476f7. 00000400:00000200:0.0:1784949039.540723:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f0571d9e 00000400:00000200:0.0:1784949039.540736:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1784949039.540737:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000200:0.0:1784949039.540753:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:1.0:1784949039.540754:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670144, portal 10 00000100:00000001:1.0:1784949039.540767:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949039.540770:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000400:00000200:0.0:1784949039.540776:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0571d9e 00000100:00000040:1.0:1784949039.540784:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949039.540786:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f0571d9e. 00000100:00000001:1.0:1784949039.540799:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949039.540803:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949039.540810:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000100:00000200:1.0:1784949039.540817:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289670144, offset 0 00000400:00000200:1.0:1784949039.540827:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949039.540828:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949039.540836:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949039.540842:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000030d4261 x1871651289670016 msgsize 224 00000400:00000200:1.0:1784949039.540859:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00100000:0.0:1784949039.540871:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:1.0:1784949039.540873:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1784949039.540883:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784949039.540892:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949039.540902:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949039.540961:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784949039.540972:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:2.0:1784949039.540978:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784949039.540980:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3558:3558:235:4294967295] 192.168.202.137@tcp LPNI seq info [2283:2283:8:4294967295] 00000100:00100000:2.0:1784949039.540986:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670016 02000000:00000001:2.0:1784949039.540996:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949039.541000:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949039.541002:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949039.541017:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949039.541021:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670016 00000020:00000001:2.0:1784949039.541027:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1784949039.541035:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000040:2.0:1784949039.541036:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784949039.541046:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949039.541054:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:0.0:1784949039.541062:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000020:00000001:2.0:1784949039.541065:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:1.0:1784949039.541083:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949039.541084:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000020:00000001:2.0:1784949039.541086:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000010:1.0:1784949039.541101:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009dcd7bb6. 00000100:00000001:0.0:1784949039.541104:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949039.541104:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949039.541116:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000200:0.0:1784949039.541126:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949039.541127:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949039.541128:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949039.541146:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000020:00000010:2.0:1784949039.541161:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000800:00000200:1.0:1784949039.541164:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949039.541180:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009dcd7bb6 type 1, nob 320 niov 1 nkiov 1 00000020:00000010:2.0:1784949039.541181:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000338c1b3a. 00000100:00000001:0.0:1784949039.541184:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1784949039.541202:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000040:0.0:1784949039.541204:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949039.541221:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949039.541223:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949039.541225:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949039.541236:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2137 usecs (3 jiffies) 00000100:00000001:2.0:1784949039.541236:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949039.541238:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949039.541251:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949039.541256:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.541257:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305978884 00000100:00000001:0.0:1784949039.541265:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949039.541278:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:0.0:1784949039.541280:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305978923 00000100:00000001:2.0:1784949039.541281:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.541293:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949039.541305:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.541310:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305978884 00000100:00000010:1.0:1784949039.541318:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:0.0:1784949039.541324:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305978884 00000100:00000001:1.0:1784949039.541327:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949039.541330:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949039.541342:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949039.541350:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305978884 00000100:00000001:0.0:1784949039.541357:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305978923 00000100:00100000:2.0:1784949039.541364:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 762 00000100:00000010:1.0:1784949039.541365:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000040:2.0:1784949039.541377:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949039.541378:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305978884 00000100:00000001:1.0:1784949039.541380:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949039.541389:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180535376 : -115539529016240 : ffff96eadb6a5c50) 00000100:00000001:1.0:1784949039.541397:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305978884 00000100:00000001:0.0:1784949039.541403:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305978923 00000100:00000040:2.0:1784949039.541404:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000030d4261 x1871651289670016/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:700/0 lens 224/0 e 0 to 0 dl 1784949050 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949039.541406:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:1.0:1784949039.541424:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949039.541427:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305978884 00000100:00000001:2.0:1784949039.541438:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949039.541439:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949039.541445:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305978923 00000020:00000010:2.0:1784949039.541449:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:1.0:1784949039.541453:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949039.541462:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949039.541469:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000001:1.0:1784949039.541471:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.541476:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305978884 00000100:00000001:1.0:1784949039.541491:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949039.541499:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000030d4261 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289670016:12345-0@lo:400: 00000100:00000001:0.0:1784949039.541500:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949039.541505:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:2.0:1784949039.541509:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670016 00000100:00000001:1.0:1784949039.541512:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949039.541515:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949039.541516:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949039.541519:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784949039.541524:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949039.541526:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949039.541529:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.541531:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.541533:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949039.541536:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949039.541543:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784949039.541552:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.541555:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949039.541556:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949039.541565:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949039.541569:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1784949039.541572:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784949039.541575:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:2.0:1784949039.541578:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784949039.541584:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784949039.541584:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.541591:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949039.541592:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949039.541599:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949039.541599:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949039.541608:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949039.541612:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949039.541617:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000cad84312. 02000000:00000001:2.0:1784949039.541622:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949039.541644:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949039.541647:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949039.541650:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670016 00010000:00000001:2.0:1784949039.541653:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949039.541659:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000030d4261 x1871651289670016/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:700/0 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949039.541669:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949039.541673:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949039.541679:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=61 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949039.541686:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949039.541694:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949039.541697:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784949039.541703:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949039.541706:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949039.541710:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949039.541713:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949039.541720:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000100:00000200:2.0:1784949039.541726:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289670016, offset 224 00000400:00000200:2.0:1784949039.541732:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949039.541742:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949039.541754:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a180 00000400:00000200:2.0:1784949039.541761:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8c65 [1] + 224 00000400:00000200:2.0:1784949039.541768:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949039.541777:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949039.541785:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949039.541794:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949039.541814:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949039.541814:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949039.541827:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009dcd7bb6. 00000800:00000001:0.0:1784949039.541837:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949039.541840:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949039.541849:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949039.541859:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949039.541860:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949039.541866:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000800:00000001:0.0:1784949039.541870:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784949039.541872:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000800:00000001:0.0:1784949039.541881:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949039.541882:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949039.541888:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949039.541888:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000100:00000001:2.0:1784949039.541899:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949039.541899:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000400:00000200:0.0:1784949039.541900:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949039.541944:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949039.541956:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949039.541967:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a200 00000100:00000001:1.0:1784949039.541979:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949039.541980:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949039.541986:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949039.541990:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949039.541993:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 166352 00010000:00000001:2.0:1784949039.541995:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000040:3.0:1784949039.541998:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949039.542001:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949039.542012:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:1.0:1784949039.542016:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:0.0:1784949039.542018:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949039.542022:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949039.542026:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000030d4261 x1871651289670016/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:700/0 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949039.542030:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949039.542033:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784949039.542037:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949039.542045:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000093aab5da. 00000400:00000200:0.0:1784949039.542048:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949039.542065:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000100:00100000:2.0:1784949039.542073:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000030d4261 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289670016:12345-0@lo:400: Request processed in 627us (1221us total) trans 0 rc 0/0 00000400:00000200:0.0:1784949039.542076:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:1.0:1784949039.542077:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000100:00000001:1.0:1784949039.542101:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949039.542104:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00100000:2.0:1784949039.542104:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 762 00000100:00000200:0.0:1784949039.542109:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:2.0:1784949039.542112:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:1.0:1784949039.542115:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949039.542124:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000010:0.0:1784949039.542125:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000825eff83. 00000100:00000001:2.0:1784949039.542133:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:0.0:1784949039.542141:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000825eff83 x1871651289670144 msgsize 224 00000020:00000010:2.0:1784949039.542145:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000338c1b3a. 00000020:00000010:2.0:1784949039.542160:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949039.542171:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 00000100:00000200:1.0:1784949039.542172:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949039.542184:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000040:2.0:1784949039.542187:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:2.0:1784949039.542205:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:2.0:1784949039.542214:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000cad84312. 00000100:00000001:1.0:1784949039.542215:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949039.542220:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949039.542223:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.542237:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949039.542237:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000800:00000001:0.0:1784949039.542240:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784949039.542244:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949039.542248:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.542250:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000001:0.0:1784949039.542255:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.542256:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949039.542261:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:1.0:1784949039.542269:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949039.542271:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949039.542276:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670144 00000400:00000010:2.0:1784949039.542278:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dee4e850. 00000100:00001000:1.0:1784949039.542280:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=61 v=5 (5 5 5 5) 02000000:00000001:3.0:1784949039.542281:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000010:2.0:1784949039.542290:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000097d07cda. 00000100:00000001:3.0:1784949039.542293:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00001000:1.0:1784949039.542299:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=61 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949039.542299:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949039.542316:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.542317:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949039.542326:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949039.542329:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670144 00010000:00000001:1.0:1784949039.542333:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949039.542335:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00010000:00000001:1.0:1784949039.542342:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949039.542342:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949039.542354:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949039.542357:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949039.542360:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949039.542365:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:1.0:1784949039.542369:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949039.542372:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949039.542389:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949039.542401:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec8cbdfe. 00000100:00000001:1.0:1784949039.542404:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949039.542408:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949039.542411:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:3.0:1784949039.542415:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec8cbdfe. 00000100:00000001:1.0:1784949039.542417:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949039.542432:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949039.542434:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000020:00000010:3.0:1784949039.542449:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000fbb8d861. 00000020:00000010:3.0:1784949039.542463:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00100000:1.0:1784949039.542467:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000001445c4ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289670016:0@lo:400: 00000100:00000001:1.0:1784949039.542482:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949039.542484:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949039.542488:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949039.542490:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949039.542496:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:1.0:1784949039.542501:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000001445c4ff x1871651289670016/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949039.542505:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949039.542525:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949039.542527:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784949039.542531:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:1.0:1784949039.542534:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1784949039.542535:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949039.542540:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:1.0:1784949039.542547:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949039.542550:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949039.542552:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949039.542554:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949039.542556:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949039.542566:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 02000000:00000001:1.0:1784949039.542571:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949039.542574:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1784949039.542576:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949039.542592:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000010:1.0:1784949039.542594:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000001445c4ff. 00000100:00000001:1.0:1784949039.542607:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949039.542616:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949039.542617:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 738 00000100:00000001:0.0:1784949039.542622:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949039.542649:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949039.542651:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949039.542653:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949039.542664:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554972544 : -115544154579072 : ffff96e9c7b5df80) 00000100:00000001:0.0:1784949039.542670:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.542672:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784949039.542677:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000825eff83 x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:700/0 lens 224/0 e 0 to 0 dl 1784949050 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949039.542680:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949039.542684:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.542696:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949039.542703:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949039.542720:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949039.542724:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec8cbdfe. 00000100:00000001:3.0:1784949039.542730:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949039.542734:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec8cbdfe. 00000100:00100000:3.0:1784949039.542741:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000825eff83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289670144:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949039.542746:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670144 00000020:00000001:3.0:1784949039.542749:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949039.542752:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949039.542754:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949039.542757:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949039.542758:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949039.542761:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949039.542764:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949039.542766:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949039.542767:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949039.542769:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949039.542771:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949039.542775:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949039.542777:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949039.542782:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000017348c4b. 02000000:00000001:3.0:1784949039.542784:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949039.542786:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949039.542788:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949039.542790:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670144 00010000:00000001:3.0:1784949039.542793:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949039.542797:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000825eff83 x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:700/0 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949039.542811:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949039.542815:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949039.542819:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=56 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949039.542825:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949039.542829:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949039.542832:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949039.542836:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949039.542838:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949039.542842:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949039.542846:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949039.542851:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003dcd3ddc. 00000100:00000200:3.0:1784949039.542858:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289670144, offset 224 00000400:00000200:3.0:1784949039.542867:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949039.542881:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949039.542890:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3559:3559:235:4294967295] 192.168.202.137@tcp LPNI seq info [2284:2284:8:4294967295] 00000400:00000200:3.0:1784949039.542939:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949039.542946:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949039.542952:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949039.542959:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949039.542967:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949039.542971:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949039.542988:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949039.542991:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949039.542994:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949039.542995:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949039.542998:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949039.543002:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000825eff83 x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:700/0 lens 224/224 e 0 to 0 dl 1784949050 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949039.543011:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000825eff83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289670144:12345-192.168.202.137@tcp:400: Request processed in 292us (870us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949039.543020:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 738 00000100:00000040:3.0:1784949039.543023:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949039.543026:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949039.543028:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949039.543033:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000fbb8d861. 00000020:00000010:3.0:1784949039.543038:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949039.543042:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000020:00000040:3.0:1784949039.543048:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949039.543051:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949039.543146:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949039.543148:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949039.543154:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000800:00000001:3.0:1784949039.543157:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949039.543160:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949039.543162:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949039.543170:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949039.543177:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949039.543188:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949039.543191:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003dcd3ddc 00000400:00000010:0.0:1784949039.543198:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003dcd3ddc. 00000400:00000200:3.0:1784949039.543205:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a200 00000100:00000001:0.0:1784949039.543209:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949039.543215:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949039.543216:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8c6d [1] + 224 00000800:00000001:3.0:1784949039.543226:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949039.543236:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949039.543239:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949039.543243:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949039.543250:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949039.543254:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949039.543265:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949039.543284:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949039.543290:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949039.543293:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.543387:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949039.543393:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949039.543401:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949039.543403:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949039.543406:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e572277. 00000400:00000200:1.0:1784949039.543413:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:1.0:1784949039.543415:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:1.0:1784949039.543419:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949039.543427:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949039.543440:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949039.543450:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949039.543452:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.543455:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949039.543457:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949039.543459:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949039.543463:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949039.543465:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949039.543472:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=56 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949039.543476:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=56 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949039.543479:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949039.543480:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949039.543483:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949039.543487:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949039.543495:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.543499:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949039.543501:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949039.543503:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949039.543504:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949039.543508:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949039.543515:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949039.543518:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949039.543520:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949039.543523:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949039.543531:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289670144:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949039.543536:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.543538:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949039.543540:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289670144/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949039.543547:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949039.543549:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949039.543551:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:1.0:1784949039.543555:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949039.543557:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949039.543558:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949039.543562:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949039.543564:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:1.0:1784949039.543567:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949039.543569:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949039.543570:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:1.0:1784949039.543574:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949039.543575:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.543577:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949039.543583:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949039.543584:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949039.543587:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949039.543589:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949039.583989:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305978923, at 4305978924 00000100:00000010:3.0:1784949039.584002:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:2.0:1784949039.584011:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305978923, at 4305978924 00000100:00000040:0.0:1784949039.584024:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305978923, at 4305978924 00000100:00000040:1.0:1784949039.584024:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305978923, at 4305978924 00000100:00000010:2.0:1784949039.584027:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:1.0:1784949039.584049:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:0.0:1784949039.584052:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00010000:00000001:2.0:1784949040.372040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949040.372051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949040.372058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949040.372060:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949040.372143:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949040.372150:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949040.372155:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949040.564065:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949040.564086:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949040.564089:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949040.564098:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949040.564102:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949040.564107:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949040.564112:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949041.396056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949041.396065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949041.396074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949041.396077:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949041.396097:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949041.396100:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949041.396104:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949041.396106:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949041.396108:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949041.396110:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949041.396253:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949041.396262:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949041.396268:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949041.587988:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949041.588002:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949041.588004:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949041.588009:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949041.588012:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949041.588016:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949041.588019:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1784949042.100058:0:343755:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.8:988 00000400:00000100:1.0:1784949042.100153:0:344067:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.12:988 00000400:00000100:2.0:1784949042.100166:0:343910:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.2:988 00000400:00000100:3.0:1784949042.100170:0:343755:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.8@tcp at host 111.222.173.8:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784949042.100213:0:343755:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.8@tcp: down 00000400:00000100:0.0:1784949042.100217:0:344068:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.14:988 00000400:00000200:3.0:1784949042.100227:0:343755:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:1.0:1784949042.100287:0:344067:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.12@tcp at host 111.222.173.12:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949042.100317:0:344067:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.12@tcp: down 00000400:00000100:2.0:1784949042.100317:0:343910:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.2@tcp at host 111.222.173.2:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:0.0:1784949042.100318:0:344068:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.14@tcp at host 111.222.173.14:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949042.100342:0:344067:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784949042.100360:0:343910:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.2@tcp: down 00000400:00000200:0.0:1784949042.100361:0:344068:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.14@tcp: down 00000800:00000100:1.0:1784949042.100372:0:344067:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000400:00000200:2.0:1784949042.100381:0:343910:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:1.0:1784949042.100383:0:344067:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:0.0:1784949042.100395:0:344068:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949042.100399:0:344067:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:2.0:1784949042.100401:0:343910:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000800:00000010:2.0:1784949042.100417:0:343910:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000800:00000100:0.0:1784949042.100418:0:344068:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000400:00000200:1.0:1784949042.100431:0:344067:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000800:00000010:0.0:1784949042.100434:0:344068:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000200:2.0:1784949042.100436:0:343910:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949042.100449:0:344068:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000100:3.0:1784949042.100455:0:343955:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.6:988 00000400:00000200:2.0:1784949042.100456:0:343910:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949042.100465:0:344067:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11315 health :900 00000400:00000200:0.0:1784949042.100475:0:344068:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949042.100493:0:344067:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949042.100502:0:343910:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949042.100503:0:344068:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949042.100511:0:344067:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000100:3.0:1784949042.100522:0:343955:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.6@tcp at host 111.222.173.6:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949042.100524:0:344068:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:2.0:1784949042.100526:0:343910:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:1.0:1784949042.100535:0:344067:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:3.0:1784949042.100551:0:343955:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.6@tcp: down 00000400:00000200:0.0:1784949042.100568:0:344068:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:2.0:1784949042.100583:0:343910:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:3.0:1784949042.100596:0:343955:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1784949042.100630:0:343955:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.6@tcp 00000800:00000010:3.0:1784949042.100634:0:343955:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:3.0:1784949042.100644:0:343955:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949042.100655:0:343955:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949042.100671:0:343955:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949042.100681:0:343955:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:3.0:1784949042.100695:0:343955:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:1.0:1784949042.100846:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949042.100865:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000100:0.0:1784949042.100868:0:343914:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.4:988 00000400:00000001:1.0:1784949042.100876:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949042.100880:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949042.100885:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784949042.100891:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000400:00000200:1.0:1784949042.100902:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949042.101021:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000100:2.0:1784949042.101022:0:343915:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.10:988 00000400:00000100:0.0:1784949042.101030:0:343914:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.4@tcp at host 111.222.173.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000010:1.0:1784949042.101032:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007db69e48. 00000400:00000200:0.0:1784949042.101052:0:343914:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.4@tcp: down 00000400:00000200:1.0:1784949042.101059:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:0.0:1784949042.101067:0:343914:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949042.101075:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000800:00000100:0.0:1784949042.101091:0:343914:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000400:00000200:1.0:1784949042.101104:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000400:00000100:2.0:1784949042.101104:0:343915:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.10@tcp at host 111.222.173.10:988 took too long: that node may be hung or experiencing high load. 00000800:00000010:0.0:1784949042.101108:0:343914:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:1.0:1784949042.101123:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000200:0.0:1784949042.101124:0:343914:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949042.101137:0:343915:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.10@tcp: down 00000400:00000001:1.0:1784949042.101147:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1784949042.101149:0:343915:0:(router.c:1888:lnet_notify()) Old news 00000400:00000001:1.0:1784949042.101157:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949042.101164:0:343914:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949042.101165:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000800:00000100:2.0:1784949042.101175:0:343915:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.10@tcp 00000400:00000010:1.0:1784949042.101178:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000800:00000010:2.0:1784949042.101190:0:343915:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f495c80e. 00000400:00000200:1.0:1784949042.101192:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949042.101194:0:343914:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949042.101215:0:343915:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949042.101220:0:343914:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:1.0:1784949042.101223:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:2.0:1784949042.101240:0:343915:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784949042.101241:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000059e5b358. 00000400:00000200:1.0:1784949042.101265:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:0.0:1784949042.101267:0:343914:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:1.0:1784949042.101292:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:2.0:1784949042.101301:0:343915:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949042.101304:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00000400:00000200:2.0:1784949042.101322:0:343915:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:1.0:1784949042.101330:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:1.0:1784949042.101337:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1784949042.101338:0:343915:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000001:1.0:1784949042.101348:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949042.101352:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000400:00000010:1.0:1784949042.101357:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000400:00000200:1.0:1784949042.101366:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949042.101388:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:1.0:1784949042.101397:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dbce179e. 00000400:00000200:1.0:1784949042.101415:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:1.0:1784949042.101423:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:1.0:1784949042.101435:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000400:00000200:1.0:1784949042.101452:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000400:00000001:1.0:1784949042.101456:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949042.101460:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949042.101463:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784949042.101467:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000400:00000200:1.0:1784949042.101472:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949042.101480:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:1.0:1784949042.101487:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000052e1e01a. 00000400:00000200:1.0:1784949042.101492:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-110 00000400:00000200:1.0:1784949042.101494:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -110 00000400:00000200:1.0:1784949042.101501:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:1.0:1784949042.101505:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:1.0:1784949042.101508:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949042.101509:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949042.101510:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000400:00000010:1.0:1784949042.101512:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000400:00000200:1.0:1784949042.101514:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949042.101515:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:1.0:1784949042.101517:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000378c7327. 00000400:00000200:1.0:1784949042.101520:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:1.0:1784949042.101522:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:1.0:1784949042.101524:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:1.0:1784949042.101527:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000001:1.0:1784949042.101529:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949042.101530:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949042.101532:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:1.0:1784949042.101534:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000400:00000200:1.0:1784949042.101535:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949042.101537:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:1.0:1784949042.101539:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000467d4d37. 00000400:00000200:1.0:1784949042.101542:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-110 00000400:00000200:1.0:1784949042.101544:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -110 00000400:00000200:1.0:1784949042.101546:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00010000:00000001:2.0:1784949042.420055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949042.420064:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949042.420072:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949042.420074:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949042.420167:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949042.420174:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949042.420178:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949042.612068:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949042.612083:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949042.612086:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949042.612091:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949042.612094:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949042.612098:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949042.612101:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949042.804062:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 000000003d48e6d9. 00000400:02000040:3.0:1784949042.804082:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000400:00000010:3.0:1784949042.807715:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 000000003d48e6d9. 00010000:00000001:2.0:1784949043.444001:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949043.444007:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949043.444015:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949043.444017:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949043.444115:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949043.444124:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949043.444130:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949043.636058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949043.636072:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949043.636074:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949043.636080:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949043.636083:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949043.636087:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949043.636090:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949043.828049:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949043.828058:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000009564dcd8. 00000400:00000010:3.0:1784949043.828068:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:3.0:1784949043.828076:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000400:00000010:3.0:1784949043.828086:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000511e5266. 00000400:00000200:3.0:1784949043.828093:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949043.828112:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949043.828121:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949043.828126:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3560:3560:241:4294967295] 192.168.202.137@tcp LPNI seq info [2285:2285:8:4294967295] 00000400:00000200:3.0:1784949043.828135:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949043.828143:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949043.828150:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949043.828156:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000800:00000200:3.0:1784949043.828164:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949043.828174:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949043.828179:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949043.828204:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11317 health :400 00000800:00000200:0.0:1784949043.828416:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949043.828425:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:0.0:1784949043.828435:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949043.828445:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:3.0:1784949043.828452:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949043.828453:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949043.828461:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949043.828470:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949043.828477:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949043.828484:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949043.828497:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949043.828512:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949043.828524:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949043.828536:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949043.828542:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3561:3561:241:4294967295] 192.168.202.137@tcp LPNI seq info [2286:2286:8:4294967295] 00000400:00000200:3.0:1784949043.828552:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949043.828558:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949043.828562:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949043.828569:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949043.828574:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949043.828577:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949043.828676:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949043.828739:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949043.828746:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000400:00000200:3.0:1784949043.828752:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949043.828756:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949043.828766:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949043.828772:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949043.828775:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949043.828780:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949043.828792:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949043.828808:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949043.828827:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8c81 00000800:00000001:0.0:1784949043.828834:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949043.828848:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949043.828851:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949043.828855:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949043.828862:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00000400:00000010:0.0:1784949043.828865:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000400:00000200:0.0:1784949043.828869:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949043.828872:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000009564dcd8. 00000400:00000010:0.0:1784949043.828875:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000800:00000001:0.0:1784949043.828883:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949043.828886:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949044.468023:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949044.468030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949044.468047:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949044.468049:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949044.468118:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949044.468125:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949044.468129:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949044.660041:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949044.660055:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949044.660059:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949044.660062:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949044.660064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949044.660071:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949044.660074:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949044.660077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.660082:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949044.660085:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000020:00000040:3.0:1784949044.660094:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949044.660100:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949044.660105:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949044.660112:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949044.660122:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000001:3.0:1784949044.660132:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949044.660143:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949044.660158:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949044.660161:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949044.660169:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_00_03+3] req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949044.660218:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949044.660223:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949044.660229:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949044.660233:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949044.660240:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949044.660245:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000031b67b62. 00000020:00000040:3.0:1784949044.660251:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949044.660255:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949044.660257:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784949044.660261:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949044.660262:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.660269:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949044.660274:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949044.660277:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784949044.660277:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000040:0.0:1784949044.660291:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949044.660292:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949044.660294:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.660318:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1784949044.660321:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949044.660324:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949044.660334:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949044.660341:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949044.660350:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949044.660356:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1784949044.660358:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289670272:0@lo:400: 00000100:00000001:3.0:1784949044.660364:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.660369:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1784949044.660381:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000031b67b62] to pc [ptlrpcd_00_00+0] req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949044.660382:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949044.660387:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949044.660395:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949044.660404:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 02000000:00000001:0.0:1784949044.660410:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949044.660421:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4e2c75c. 00000400:00000010:0.0:1784949044.660430:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000200:0.0:1784949044.660439:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670272, portal 25 00000100:00000001:0.0:1784949044.660443:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949044.660446:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554960640 : -115544154590976 : ffff96e9c7b5b100) 00000100:00000001:3.0:1784949044.660451:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949044.660461:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949044.660468:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11321) 00000100:00000001:2.0:1784949044.660478:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949044.660479:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949044.660488:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949044.660490:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949044.660495:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:0.0:1784949044.660504:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000040:2.0:1784949044.660508:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949044.660512:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289670272, offset 0 00000100:00000001:3.0:1784949044.660518:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949044.660530:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949044.660533:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784949044.660535:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949044.660541:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949044.660547:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949044.660551:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949044.660559:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949044.660560:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:2.0:1784949044.660569:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289670400:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949044.660578:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949044.660612:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:0.0:1784949044.660615:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a280 02000000:00000001:2.0:1784949044.660616:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949044.660625:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949044.660634:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007a6587e5. 00000400:00000200:0.0:1784949044.660642:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8265 [2] + 0 02000000:00000001:2.0:1784949044.660643:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949044.660653:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000aee1f7f7. 00000400:00000010:0.0:1784949044.660657:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000045b97c3a. 00000400:00000200:0.0:1784949044.660669:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000be5664ea 00000400:00000010:2.0:1784949044.660680:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000400:00000200:0.0:1784949044.660681:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784949044.660693:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670400, portal 10 00000400:00000200:0.0:1784949044.660697:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949044.660701:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949044.660708:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554959744 : -115544154591872 : ffff96e9c7b5ad80) 00000400:00000200:0.0:1784949044.660713:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000be5664ea 00000100:00000040:2.0:1784949044.660721:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949044.660724:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000be5664ea. 00000100:00000001:0.0:1784949044.660738:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784949044.660740:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:0.0:1784949044.660744:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949044.660751:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000bcf1bd0a x1871651289670272 msgsize 224 00000100:00000040:2.0:1784949044.660757:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949044.660766:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00100000:0.0:1784949044.660767:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000200:2.0:1784949044.660773:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289670400, offset 0 00000100:00000040:0.0:1784949044.660774:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949044.660790:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949044.660800:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949044.660804:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949044.660815:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949044.660820:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949044.660826:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949044.660830:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949044.660838:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3562:3562:241:4294967295] 192.168.202.137@tcp LPNI seq info [2287:2287:8:4294967295] 00000100:00000001:1.0:1784949044.660846:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784949044.660860:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949044.660871:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949044.660875:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000800:00000010:2.0:1784949044.660879:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002f91e44d. 00000400:00000010:0.0:1784949044.660882:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784949044.660895:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:2.0:1784949044.660896:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784949044.660960:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949044.660963:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670272 02000000:00000001:1.0:1784949044.660974:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000200:2.0:1784949044.660985:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949044.660987:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949044.661005:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949044.661007:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002f91e44d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949044.661010:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784949044.661028:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949044.661030:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949044.661039:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949044.661045:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670272 00000100:00000040:2.0:1784949044.661054:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2138 usecs (3 jiffies) 00000020:00000001:1.0:1784949044.661058:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949044.661068:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305984004 00000020:00000040:1.0:1784949044.661069:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949044.661077:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949044.661082:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949044.661085:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:2.0:1784949044.661087:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a2179890. 00000020:00000040:1.0:1784949044.661088:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949044.661098:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949044.661104:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949044.661107:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305984043 00000100:00000001:2.0:1784949044.661111:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000001:1.0:1784949044.661113:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949044.661130:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949044.661133:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305984004 00000100:00000001:0.0:1784949044.661140:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305984004 00000100:00000010:2.0:1784949044.661141:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000010de4e21. 00000100:00000001:2.0:1784949044.661157:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949044.661161:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305984043 00000100:00000001:2.0:1784949044.661174:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305984004 00000100:00000010:2.0:1784949044.661183:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e2039f57. 00000100:00000001:0.0:1784949044.661184:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305984004 00000100:00000001:2.0:1784949044.661190:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949044.661203:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305984043 00000100:00000001:2.0:1784949044.661212:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305984004 00000100:00000001:0.0:1784949044.661221:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305984004 00000100:00000010:2.0:1784949044.661223:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000100:00000001:0.0:1784949044.661237:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305984043 00000100:00000001:2.0:1784949044.661238:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949044.661263:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949044.661271:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305984004 00000100:00000001:2.0:1784949044.661273:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949044.661288:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.661293:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949044.661296:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949044.661300:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.661310:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949044.661310:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949044.661319:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949044.661320:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949044.661335:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949044.661340:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949044.661342:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949044.661349:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949044.661351:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.661362:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949044.661363:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.661369:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:3.0:1784949044.661375:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949044.661377:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.661391:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949044.661392:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002f91e44d. 00000400:00000200:3.0:1784949044.661405:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949044.661419:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949044.661420:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949044.661431:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000800:00000001:2.0:1784949044.661436:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949044.661437:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000020:00000010:0.0:1784949044.661439:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000800:00000001:2.0:1784949044.661446:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949044.661454:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949044.661456:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:2.0:1784949044.661460:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949044.661462:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000100:00000200:3.0:1784949044.661474:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949044.661477:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000010:0.0:1784949044.661478:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000020:00000010:0.0:1784949044.661492:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007194bf82. 00000020:00000010:0.0:1784949044.661506:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:2.0:1784949044.661508:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a300 00000100:00000001:3.0:1784949044.661509:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949044.661523:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949044.661538:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:2.0:1784949044.661541:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 166576 00000100:00000001:3.0:1784949044.661541:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949044.661546:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949044.661550:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949044.661552:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949044.661569:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949044.661574:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949044.661609:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949044.661619:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949044.661631:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949044.661633:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949044.661646:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949044.661659:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949044.661663:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949044.661665:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949044.661672:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000010:2.0:1784949044.661676:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cb70416b. 00000100:00100000:0.0:1784949044.661682:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 763 00000100:00000040:2.0:1784949044.661688:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000cb70416b x1871651289670400 msgsize 224 00000100:00000040:0.0:1784949044.661690:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949044.661700:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180531280 : -115539529020336 : ffff96eadb6a4c50) 00000100:00000040:0.0:1784949044.661715:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bcf1bd0a x1871651289670272/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:705/0 lens 224/0 e 0 to 0 dl 1784949055 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949044.661717:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949044.661734:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949044.661742:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000100:00000001:2.0:1784949044.661748:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949044.661750:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949044.661757:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000100:00100000:0.0:1784949044.661769:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bcf1bd0a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289670272:12345-0@lo:400: 00000800:00000001:2.0:1784949044.661771:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784949044.661779:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670272 00000020:00000001:0.0:1784949044.661787:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:2.0:1784949044.661792:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949044.661794:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949044.661800:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949044.661807:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949044.661812:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949044.661815:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:0.0:1784949044.661819:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949044.661825:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949044.661825:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670400 00000020:00000001:0.0:1784949044.661832:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000001:3.0:1784949044.661835:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:0.0:1784949044.661838:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949044.661844:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:0.0:1784949044.661846:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949044.661852:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949044.661853:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949044.661861:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949044.661863:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949044.661868:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:3.0:1784949044.661871:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670400 00000020:00000001:3.0:1784949044.661879:0:342764:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000010:0.0:1784949044.661880:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e83d824c. 00000020:00000040:3.0:1784949044.661885:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000001:0.0:1784949044.661887:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949044.661893:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949044.661894:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949044.661954:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949044.661956:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000040:0.0:1784949044.661962:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670272 00000020:00000001:3.0:1784949044.661963:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:0.0:1784949044.661970:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:3.0:1784949044.661972:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949044.661981:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000200:0.0:1784949044.661983:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bcf1bd0a x1871651289670272/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:705/0 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949044.661989:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:3.0:1784949044.662000:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1784949044.662002:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:3.0:1784949044.662006:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00010000:00000001:0.0:1784949044.662008:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949044.662015:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=66 v=5 (1 1 1 1) 00000020:00000010:3.0:1784949044.662017:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000100:00000001:0.0:1784949044.662027:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1784949044.662029:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000092bc93ba. 00000100:00000040:0.0:1784949044.662033:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000010:3.0:1784949044.662041:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949044.662042:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949044.662053:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949044.662056:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949044.662060:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949044.662061:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949044.662069:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949044.662070:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949044.662075:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000010:0.0:1784949044.662076:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000001:3.0:1784949044.662079:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949044.662086:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289670272, offset 224 00000100:00000001:3.0:1784949044.662090:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:0.0:1784949044.662097:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949044.662098:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949044.662103:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949044.662111:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784949044.662120:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a280 00000100:00000001:3.0:1784949044.662123:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949044.662132:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8c85 [1] + 224 00000400:00000200:0.0:1784949044.662143:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949044.662146:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784949044.662149:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949044.662152:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949044.662158:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1784949044.662165:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 739 00000100:00000200:0.0:1784949044.662170:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949044.662173:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949044.662182:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562020352 : -115544147531264 : ffff96e9c8216a00) 00000100:00000040:3.0:1784949044.662198:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000cb70416b x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:705/0 lens 224/0 e 0 to 0 dl 1784949055 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949044.662202:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949044.662255:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949044.662261:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:3.0:1784949044.662264:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949044.662267:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000100:00000001:0.0:1784949044.662274:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1784949044.662276:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000cb70416b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289670400:12345-192.168.202.137@tcp:400: 00000400:00000200:0.0:1784949044.662283:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000100:00000200:3.0:1784949044.662288:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670400 00000400:00000010:0.0:1784949044.662289:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784949044.662298:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:3.0:1784949044.662302:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949044.662303:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:3.0:1784949044.662315:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949044.662316:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949044.662324:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:3.0:1784949044.662326:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949044.662329:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949044.662335:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949044.662338:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949044.662341:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:3.0:1784949044.662347:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000040:0.0:1784949044.662354:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bcf1bd0a x1871651289670272/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:705/0 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949044.662361:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949044.662370:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949044.662372:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949044.662382:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bcf1bd0a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289670272:12345-0@lo:400: Request processed in 639us (1621us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949044.662384:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949044.662387:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949044.662398:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949044.662406:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949044.662408:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1784949044.662416:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:0.0:1784949044.662419:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 763 00000400:00000001:1.0:1784949044.662419:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949044.662423:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949044.662435:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000010:1.0:1784949044.662436:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4e2c75c. 00000100:00000040:0.0:1784949044.662437:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 02000000:00000010:3.0:1784949044.662449:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b34d2015. 00000100:00000001:0.0:1784949044.662455:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:3.0:1784949044.662457:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949044.662459:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000100:00000001:0.0:1784949044.662462:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000010:1.0:1784949044.662475:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:3.0:1784949044.662477:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949044.662479:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007194bf82. 00000020:00000001:3.0:1784949044.662504:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949044.662508:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949044.662512:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000040:3.0:1784949044.662512:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670400 00000020:00000010:0.0:1784949044.662517:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00010000:00000001:3.0:1784949044.662530:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000040:0.0:1784949044.662532:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:1.0:1784949044.662536:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949044.662547:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000200:3.0:1784949044.662550:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000cb70416b x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:705/0 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949044.662556:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000e83d824c. 02000000:00000001:0.0:1784949044.662571:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949044.662580:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949044.662628:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:0.0:1784949044.662649:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009b268f62. 00000100:00000200:1.0:1784949044.662659:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949044.662662:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000010:0.0:1784949044.662664:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000021ac459d. 00010000:00000001:3.0:1784949044.662673:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949044.662693:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=61 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949044.662713:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949044.662713:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949044.662730:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949044.662734:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949044.662740:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:3.0:1784949044.662741:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949044.662751:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:3.0:1784949044.662751:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784949044.662764:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:3.0:1784949044.662765:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949044.662773:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949044.662776:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949044.662779:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949044.662790:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00001000:1.0:1784949044.662794:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=66 v=5 (5 5 5 5) 00000400:00000010:3.0:1784949044.662797:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000100:00001000:1.0:1784949044.662815:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=66 v=5 (1 1 1 1) 00000100:00000200:3.0:1784949044.662815:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289670400, offset 224 00000100:00000001:1.0:1784949044.662840:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949044.662845:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949044.662849:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00010000:00000001:1.0:1784949044.662855:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949044.662860:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949044.662870:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949044.662870:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000040:1.0:1784949044.662898:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949044.662917:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949044.662947:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3563:3563:241:4294967295] 192.168.202.137@tcp LPNI seq info [2288:2288:8:4294967295] 00000400:00000200:3.0:1784949044.662967:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949044.662984:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949044.662990:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949044.662992:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949044.663010:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000100:00000040:1.0:1784949044.663011:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949044.663043:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:1.0:1784949044.663052:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289670272:0@lo:400: 00000800:00000200:3.0:1784949044.663066:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949044.663067:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949044.663081:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949044.663082:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949044.663094:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289670272/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949044.663104:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949044.663109:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1784949044.663109:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949044.663118:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 00000100:00000040:3.0:1784949044.663123:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949044.663132:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 02000000:00000001:1.0:1784949044.663134:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949044.663137:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949044.663140:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:3.0:1784949044.663148:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000040:1.0:1784949044.663151:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949044.663160:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949044.663167:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 00000100:00000040:3.0:1784949044.663169:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cb70416b x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:705/0 lens 224/224 e 0 to 0 dl 1784949055 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949044.663184:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949044.663195:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949044.663207:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000100:00100000:3.0:1784949044.663221:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000cb70416b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289670400:12345-192.168.202.137@tcp:400: Request processed in 953us (1519us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949044.663224:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949044.663229:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949044.663248:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949044.663260:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 739 00000100:00000001:1.0:1784949044.663262:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949044.663267:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949044.663267:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000800:00000200:0.0:1784949044.663275:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949044.663280:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949044.663286:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000010:0.0:1784949044.663287:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000100:00000001:1.0:1784949044.663288:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949044.663301:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:0.0:1784949044.663305:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949044.663312:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000092bc93ba. 00000800:00000001:2.0:1784949044.663318:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949044.663326:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949044.663330:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000400:00000200:0.0:1784949044.663333:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00000800:00000001:2.0:1784949044.663335:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949044.663341:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000800:00000001:2.0:1784949044.663343:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784949044.663344:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000100:00000001:0.0:1784949044.663352:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784949044.663353:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784949044.663358:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949044.663359:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949044.663372:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949044.663372:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949044.663396:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a300 00000400:00000200:2.0:1784949044.663405:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8c8d [1] + 224 00000800:00000001:2.0:1784949044.663412:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949044.663425:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949044.663429:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949044.663433:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949044.663439:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949044.663444:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949044.663456:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949044.663481:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949044.663488:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949044.663492:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949044.663536:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949044.663542:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949044.663549:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949044.663552:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949044.663554:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000aee1f7f7. 00000400:00000200:3.0:1784949044.663560:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:3.0:1784949044.663563:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000100:00000001:3.0:1784949044.663566:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949044.663573:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949044.663614:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949044.663623:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949044.663626:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949044.663629:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949044.663633:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949044.663635:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949044.663638:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949044.663641:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949044.663648:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=61 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949044.663652:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=61 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949044.663656:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949044.663658:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949044.663661:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949044.663666:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949044.663674:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949044.663680:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949044.663682:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949044.663684:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949044.663686:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949044.663691:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949044.663698:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949044.663702:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949044.663704:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949044.663707:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949044.663715:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289670400:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949044.663721:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949044.663724:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949044.663727:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000031b67b62 x1871651289670400/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949044.663735:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949044.663737:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949044.663740:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007a6587e5. 02000000:00000001:3.0:1784949044.663746:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949044.663748:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949044.663750:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949044.663752:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949044.663754:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 02000000:00000001:3.0:1784949044.663758:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949044.663760:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949044.663761:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000031b67b62. 00000100:00000001:3.0:1784949044.663766:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949044.663767:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949044.663770:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949044.663775:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949044.663777:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949044.663779:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949044.663782:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949044.704043:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305984043, at 4305984044 00000100:00000010:0.0:1784949044.704060:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a2179890. 00000100:00000040:3.0:1784949044.704097:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305984043, at 4305984044 00000100:00000010:3.0:1784949044.704112:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00000100:00000040:1.0:1784949044.704125:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305984043, at 4305984044 00000100:00000040:2.0:1784949044.704126:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305984043, at 4305984044 00000100:00000010:1.0:1784949044.704149:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000010de4e21. 00000100:00000010:2.0:1784949044.704158:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00010000:00000001:2.0:1784949045.492008:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949045.492014:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949045.492022:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949045.492024:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949045.492136:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949045.492147:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949045.492152:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949045.684022:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949045.684042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949045.684045:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949045.684052:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949045.684055:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949045.684059:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949045.684061:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949046.515998:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949046.516021:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949046.516029:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949046.516031:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949046.516152:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949046.516159:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949046.516163:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949046.708221:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949046.708234:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949046.708237:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949046.708241:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949046.708245:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949046.708249:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949046.708252:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949047.540044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949047.540051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949047.540058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949047.540061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949047.540137:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949047.540148:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949047.540154:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949047.732034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949047.732046:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949047.732049:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949047.732054:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949047.732057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949047.732060:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949047.732063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949048.564012:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949048.564019:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949048.564026:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949048.564028:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949048.564159:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949048.564165:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949048.564170:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949048.756030:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949048.756044:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949048.756047:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949048.756052:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949048.756055:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949048.756060:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949048.756064:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949049.588038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949049.588048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949049.588056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949049.588057:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949049.588115:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949049.588123:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949049.588127:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1784949049.780015:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949049.780026:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949049.780037:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000031b67b62. 00000020:00000040:3.0:1784949049.780047:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949049.780055:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949049.780060:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949049.780067:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949049.780078:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:3.0:1784949049.780087:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949049.780101:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949049.780118:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949049.780122:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949049.780122:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.780138:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784949049.780139:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000031b67b62] to pc [ptlrpcd_00_01+1] req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949049.780144:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949049.780155:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949049.780161:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949049.780169:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.780174:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949049.780201:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949049.780212:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949049.780219:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949049.780226:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949049.780233:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949049.780239:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005c8e6480. 00000020:00000040:3.0:1784949049.780246:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949049.780252:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949049.780255:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949049.780259:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949049.780266:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:3.0:1784949049.780274:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.780277:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949049.780283:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784949049.780288:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949049.780290:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949049.780308:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949049.780322:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949049.780328:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.780344:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1784949049.780347:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c8e6480] to pc [ptlrpcd_00_02+2] req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949049.780351:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.780361:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949049.780364:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949049.780371:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949049.780375:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949049.780387:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949049.780395:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289670528:0@lo:400: 00000100:00000001:0.0:1784949049.780403:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949049.780414:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949049.780417:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949049.780420:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949049.780425:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949049.780429:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949049.780436:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949049.780445:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11326) 00000400:00000010:0.0:1784949049.780446:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e6551ee. 00000100:00000001:2.0:1784949049.780453:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784949049.780460:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000001:2.0:1784949049.780462:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949049.780471:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000200:0.0:1784949049.780477:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670528, portal 25 00000100:00000040:2.0:1784949049.780486:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949049.780487:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949049.780493:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554959744 : -115544154591872 : ffff96e9c7b5ad80) 00000100:00000040:0.0:1784949049.780513:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949049.780516:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949049.780523:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949049.780524:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949049.780531:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949049.780536:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949049.780587:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949049.780592:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949049.780597:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000100:00000200:0.0:1784949049.780602:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289670528, offset 0 00000400:00000200:0.0:1784949049.780615:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784949049.780639:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784949049.780654:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a380 00000400:00000200:0.0:1784949049.780668:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x828d [2] + 0 00000400:00000010:0.0:1784949049.780674:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e686bcb0. 00000400:00000200:0.0:1784949049.780683:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000105270ec 00000400:00000200:0.0:1784949049.780693:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949049.780700:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949049.780707:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000105270ec 00000400:00000010:0.0:1784949049.780711:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000105270ec. 00000100:00000001:0.0:1784949049.780718:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949049.780721:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949049.780725:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000012145ed7 x1871651289670528 msgsize 224 00000100:00100000:0.0:1784949049.780731:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949049.780733:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949049.780759:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949049.780770:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:0.0:1784949049.780778:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 02000000:00000001:2.0:1784949049.780779:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949049.780790:0:337120:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1784949049.780792:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949049.780804:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949049.780806:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005c8e6480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289670656:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949049.780824:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949049.780831:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:2.0:1784949049.780835:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949049.780837:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1784949049.780838:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670528 02000000:00000001:2.0:1784949049.780842:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949049.780847:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:0.0:1784949049.780849:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949049.780850:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949049.780854:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949049.780861:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949049.780863:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 02000000:00000001:3.0:1784949049.780872:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949049.780875:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949049.780885:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670528 00000100:00000001:0.0:1784949049.780888:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949049.780889:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b5489cc4. 00000020:00000001:3.0:1784949049.780892:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949049.780900:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949049.780941:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949049.780954:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.780963:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305989163 00000100:00000010:0.0:1784949049.780971:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000020:00000001:3.0:1784949049.780976:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949049.780982:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949049.780985:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1784949049.780987:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000010:2.0:1784949049.780990:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000020:00000001:3.0:1784949049.780996:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949049.780997:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305989163 00000100:00000200:2.0:1784949049.781006:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670656, portal 10 00000100:00000001:3.0:1784949049.781009:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:0.0:1784949049.781010:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:2.0:1784949049.781020:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000010:3.0:1784949049.781020:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:0.0:1784949049.781023:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949049.781032:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554963328 : -115544154588288 : ffff96e9c7b5bb80) 00000100:00000001:3.0:1784949049.781034:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949049.781037:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305989163 00000020:00000010:3.0:1784949049.781046:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000100:00000010:0.0:1784949049.781050:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000040:2.0:1784949049.781059:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949049.781060:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000fff5e130. 00000100:00000001:0.0:1784949049.781061:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949049.781070:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305989163 00000020:00000010:3.0:1784949049.781075:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c6df765c. 00000100:00000010:0.0:1784949049.781081:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000020:00000010:3.0:1784949049.781091:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949049.781093:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949049.781104:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949049.781110:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949049.781115:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949049.781120:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949049.781124:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949049.781126:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949049.781127:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949049.781134:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949049.781137:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949049.781138:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949049.781149:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949049.781150:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000001:0.0:1784949049.781153:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949049.781161:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000200:2.0:1784949049.781164:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289670656, offset 0 00000100:00000001:0.0:1784949049.781167:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949049.781173:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949049.781175:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949049.781182:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949049.781184:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949049.781193:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949049.781204:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784949049.781212:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949049.781214:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949049.781228:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:3.0:1784949049.781238:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 764 00000400:00000200:2.0:1784949049.781242:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949049.781251:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1784949049.781253:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:2.0:1784949049.781258:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949049.781265:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180539472 : -115539529012144 : ffff96eadb6a6c50) 00000400:00000200:2.0:1784949049.781268:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3564:3564:241:4294967295] 192.168.202.137@tcp LPNI seq info [2289:2289:8:4294967295] 00000100:00000040:3.0:1784949049.781288:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000012145ed7 x1871651289670528/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:710/0 lens 224/0 e 0 to 0 dl 1784949060 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949049.781299:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949049.781325:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:2.0:1784949049.781327:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000010:3.0:1784949049.781333:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000800:00000010:2.0:1784949049.781337:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f95b4924. 00000100:00000001:3.0:1784949049.781342:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949049.781350:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000800:00000200:2.0:1784949049.781357:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:3.0:1784949049.781370:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000012145ed7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289670528:12345-0@lo:400: 00000800:00000200:2.0:1784949049.781381:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000200:3.0:1784949049.781384:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670528 00000800:00000200:2.0:1784949049.781392:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f95b4924 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:3.0:1784949049.781394:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949049.781403:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949049.781407:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949049.781411:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949049.781414:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949049.781420:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949049.781423:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949049.781428:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949049.781434:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949049.781439:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.781440:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949049.781448:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:2.0:1784949049.781451:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2141 usecs (3 jiffies) 00000100:00000001:3.0:1784949049.781458:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784949049.781459:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305989124 00000100:00000001:2.0:1784949049.781470:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305989163 02000000:00000001:3.0:1784949049.781471:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949049.781475:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305989124 00000100:00000001:2.0:1784949049.781481:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305989163 02000000:00000010:3.0:1784949049.781484:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007617a2e3. 00000100:00000001:2.0:1784949049.781485:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305989124 00000100:00000001:2.0:1784949049.781492:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305989163 02000000:00000001:3.0:1784949049.781496:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.781498:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305989124 00000100:00000001:2.0:1784949049.781510:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305989163 00000100:00000001:3.0:1784949049.781511:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.781513:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949049.781523:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.781524:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949049.781538:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670528 00000100:00000001:2.0:1784949049.781566:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.781574:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949049.781577:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949049.781588:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000200:3.0:1784949049.781591:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000012145ed7 x1871651289670528/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:710/0 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949049.781595:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949049.781607:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1784949049.781613:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949049.781614:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1784949049.781621:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.781625:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949049.781633:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=71 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949049.781636:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949049.781654:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1784949049.781669:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:3.0:1784949049.781669:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000010:0.0:1784949049.781683:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f95b4924. 00000100:00000001:3.0:1784949049.781683:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949049.781697:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:0.0:1784949049.781698:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784949049.781705:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949049.781718:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949049.781721:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949049.781730:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000010:0.0:1784949049.781737:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000040:3.0:1784949049.781740:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:2.0:1784949049.781750:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949049.781751:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:3.0:1784949049.781754:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003dcd3ddc. 00000100:00000200:3.0:1784949049.781769:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289670528, offset 224 00000100:00000200:0.0:1784949049.781770:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949049.781775:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949049.781787:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949049.781800:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949049.781801:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949049.781807:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949049.781823:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949049.781827:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949049.781842:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949049.781857:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949049.781858:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a400 00000100:00000001:0.0:1784949049.781867:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784949049.781884:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a380 00000400:00000200:2.0:1784949049.781885:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 166800 00000800:00000001:2.0:1784949049.781951:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949049.781958:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8ca1 [1] + 224 00000800:00000001:2.0:1784949049.781964:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949049.781971:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949049.781972:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949049.781984:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949049.781986:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949049.781997:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949049.782003:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949049.782004:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:2.0:1784949049.782015:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000073fe9b92. 00000100:00000200:3.0:1784949049.782020:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949049.782022:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000073fe9b92 x1871651289670656 msgsize 224 00000100:00100000:2.0:1784949049.782044:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:3.0:1784949049.782064:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949049.782069:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:2.0:1784949049.782082:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949049.782091:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949049.782122:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949049.782128:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949049.782133:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670656 02000000:00000001:0.0:1784949049.782136:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949049.782139:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949049.782142:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949049.782142:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003dcd3ddc 02000000:00000001:0.0:1784949049.782150:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949049.782152:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003dcd3ddc. 00000100:00100000:0.0:1784949049.782157:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670656 00000020:00000001:0.0:1784949049.782163:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949049.782165:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000040:0.0:1784949049.782167:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:3.0:1784949049.782172:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:0.0:1784949049.782173:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949049.782182:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949049.782186:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949049.782189:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949049.782192:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949049.782200:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:3.0:1784949049.782204:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000010:0.0:1784949049.782208:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00010000:00000001:3.0:1784949049.782214:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949049.782217:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949049.782222:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949049.782227:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:1.0:1784949049.782227:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949049.782233:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949049.782240:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:0.0:1784949049.782250:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001879354a. 00000100:00000040:3.0:1784949049.782251:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000012145ed7 x1871651289670528/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:710/0 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949049.782259:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:0.0:1784949049.782266:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007194bf82. 00000400:00000001:1.0:1784949049.782272:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949049.782282:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e6551ee. 00000020:00000010:0.0:1784949049.782286:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:1.0:1784949049.782301:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:1.0:1784949049.782308:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000040:0.0:1784949049.782320:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00100000:3.0:1784949049.782320:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000012145ed7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289670528:12345-0@lo:400: Request processed in 981us (1577us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949049.782322:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949049.782337:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000200:1.0:1784949049.782347:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949049.782348:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:3.0:1784949049.782359:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 764 00000100:00000001:0.0:1784949049.782361:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949049.782380:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949049.782382:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949049.782397:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949049.782398:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:1.0:1784949049.782406:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949049.782409:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949049.782411:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949049.782424:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c6df765c. 00000100:00000001:0.0:1784949049.782442:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949049.782444:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949049.782450:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784949049.782459:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000fff5e130. 00000100:00000001:1.0:1784949049.782462:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949049.782470:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000040:3.0:1784949049.782475:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784949049.782476:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:1.0:1784949049.782477:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:3.0:1784949049.782484:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949049.782496:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949049.782498:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000010:3.0:1784949049.782498:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007617a2e3. 00000100:00000001:1.0:1784949049.782516:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949049.782518:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00100000:0.0:1784949049.782530:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 740 02000000:00000001:1.0:1784949049.782531:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949049.782532:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949049.782571:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000200:3.0:1784949049.782584:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784949049.782594:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562017664 : -115544147533952 : ffff96e9c8215f80) 00000100:00001000:1.0:1784949049.782601:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=71 v=5 (5 5 5 5) 00000400:00000010:3.0:1784949049.782602:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000df521cee. 00000100:00000040:0.0:1784949049.782606:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000073fe9b92 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:710/0 lens 224/0 e 0 to 0 dl 1784949060 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949049.782623:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bfafda61. 00000100:00001000:1.0:1784949049.782628:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=71 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949049.782653:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949049.782665:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.782667:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:1.0:1784949049.782679:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:0.0:1784949049.782680:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00010000:00000001:1.0:1784949049.782684:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.782693:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949049.782693:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949049.782709:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000040:1.0:1784949049.782715:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949049.782733:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000073fe9b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289670656:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949049.782762:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670656 00000020:00000001:0.0:1784949049.782773:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949049.782781:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949049.782788:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949049.782802:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949049.782811:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784949049.782812:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949049.782828:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949049.782829:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949049.782837:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949049.782839:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949049.782856:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784949049.782863:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949049.782865:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949049.782875:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949049.782889:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949049.782899:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00100000:1.0:1784949049.782904:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289670528:0@lo:400: 00000100:00000001:1.0:1784949049.782913:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949049.782941:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949049.782946:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e83d824c. 02000000:00000001:0.0:1784949049.782950:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.782954:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949049.782957:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949049.782960:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670656 00010000:00000001:0.0:1784949049.782964:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949049.782964:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000200:0.0:1784949049.782972:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000073fe9b92 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:710/0 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949049.782981:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000031b67b62 x1871651289670528/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949049.782994:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949049.783002:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949049.783011:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=66 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949049.783029:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949049.783033:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:0.0:1784949049.783035:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949049.783043:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949049.783045:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1784949049.783048:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000010:1.0:1784949049.783052:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949049.783053:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949049.783070:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949049.783071:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949049.783079:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000040:0.0:1784949049.783081:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000040:1.0:1784949049.783088:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000010:0.0:1784949049.783089:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000020:00000001:1.0:1784949049.783108:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000200:0.0:1784949049.783110:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289670656, offset 224 02000000:00000010:1.0:1784949049.783115:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 00000400:00000200:0.0:1784949049.783124:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949049.783126:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949049.783131:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949049.783134:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000031b67b62. 00000400:00000200:0.0:1784949049.783143:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949049.783146:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949049.783160:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949049.783167:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3565:3565:241:4294967295] 192.168.202.137@tcp LPNI seq info [2290:2290:8:4294967295] 00000100:00000001:1.0:1784949049.783170:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949049.783192:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949049.783203:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949049.783209:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949049.783212:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949049.783226:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949049.783238:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949049.783248:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a3cf4688. 00000800:00000200:0.0:1784949049.783259:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949049.783269:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949049.783275:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a3cf4688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949049.783302:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949049.783309:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949049.783313:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949049.783316:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949049.783320:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949049.783329:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000073fe9b92 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:710/0 lens 224/224 e 0 to 0 dl 1784949060 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949049.783348:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000073fe9b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289670656:12345-192.168.202.137@tcp:400: Request processed in 680us (1323us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949049.783361:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 740 00000100:00000040:0.0:1784949049.783366:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949049.783369:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949049.783371:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949049.783377:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007194bf82. 00000020:00000010:0.0:1784949049.783382:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949049.783386:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001879354a. 00000020:00000040:0.0:1784949049.783391:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949049.783395:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949049.783491:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949049.783492:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784949049.783501:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a3cf4688. 00000800:00000001:0.0:1784949049.783507:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949049.783510:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949049.783517:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949049.783526:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949049.783527:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949049.783537:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000200:0.0:1784949049.783570:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:2.0:1784949049.783573:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000100:00000001:2.0:1784949049.783585:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:0.0:1784949049.783590:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a400 00000100:00000001:2.0:1784949049.783593:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784949049.783605:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8ca9 [1] + 224 00000800:00000001:0.0:1784949049.783613:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949049.783627:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949049.783630:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949049.783636:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949049.783642:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949049.783646:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949049.783658:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949049.783680:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949049.783687:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949049.783690:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949049.783732:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949049.783738:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949049.783744:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949049.783747:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949049.783749:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b5489cc4. 00000400:00000200:2.0:1784949049.783755:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000010:2.0:1784949049.783757:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000100:00000001:2.0:1784949049.783761:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949049.783768:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949049.783781:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949049.783791:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949049.783793:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949049.783796:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949049.783799:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949049.783802:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949049.783804:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949049.783807:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949049.783814:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=66 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949049.783818:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=66 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949049.783821:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949049.783823:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949049.783826:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949049.783831:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949049.783840:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.783843:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949049.783846:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949049.783848:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949049.783850:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949049.783853:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949049.783862:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949049.783866:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949049.783868:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949049.783872:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949049.783882:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005c8e6480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289670656:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949049.783887:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949049.783890:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949049.783892:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005c8e6480 x1871651289670656/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949049.783900:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949049.783903:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949049.783905:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 02000000:00000001:2.0:1784949049.783939:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949049.783942:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949049.783944:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949049.783947:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949049.783949:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:2.0:1784949049.783954:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949049.783956:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949049.783957:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005c8e6480. 00000100:00000001:2.0:1784949049.783961:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949049.783963:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949049.783965:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949049.783970:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949049.783972:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949049.783975:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949049.783977:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949049.824001:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305989163, at 4305989164 00000100:00000010:3.0:1784949049.824016:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000040:0.0:1784949049.824055:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305989163, at 4305989164 00000100:00000040:2.0:1784949049.824068:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305989163, at 4305989164 00000100:00000010:0.0:1784949049.824071:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:1.0:1784949049.824085:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305989163, at 4305989164 00000100:00000010:2.0:1784949049.824087:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:1.0:1784949049.824107:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00010000:00000001:2.0:1784949050.612060:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949050.612070:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949050.612080:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949050.612083:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949050.612184:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949050.612190:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949050.612194:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949050.804001:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949050.804015:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949050.804017:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949050.804022:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949050.804026:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949050.804029:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949050.804031:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949051.636011:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949051.636020:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949051.636027:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949051.636030:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949051.636033:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949051.636036:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949051.636039:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949051.636041:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949051.636043:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949051.636045:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949051.636101:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949051.636107:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949051.636111:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949051.827985:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949051.827999:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949051.828001:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949051.828006:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949051.828009:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949051.828012:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949051.828015:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949052.660056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949052.660066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949052.660073:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949052.660075:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949052.660195:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949052.660205:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949052.660211:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949052.852181:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949052.852194:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949052.852197:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949052.852204:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949052.852209:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949052.852213:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949052.852216:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949053.684051:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949053.684061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949053.684069:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949053.684071:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949053.684202:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949053.684213:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949053.684219:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949053.876031:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949053.876044:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949053.876047:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949053.876053:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949053.876057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949053.876062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949053.876065:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1784949054.388046:0:344013:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.27:988 00000400:00000100:1.0:1784949054.388121:0:344013:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.27@tcp at host 111.222.173.27:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949054.388135:0:344013:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.27@tcp: down 00000800:00000200:1.0:1784949054.388148:0:344013:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.173.27@tcp down 00010000:00000001:2.0:1784949054.708062:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949054.708071:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949054.708079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949054.708081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949054.708183:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949054.708190:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949054.708195:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949054.900040:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1784949054.900047:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949054.900050:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1784949054.900060:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949054.900070:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005c8e6480. 00000100:00100000:0.0:1784949054.900071:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949029/real 1784949029] req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949054 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949054.900082:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949054.900095:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949054.900100:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:0.0:1784949054.900105:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1784949054.900108:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784949054.900111:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949054.900120:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000062d113c0. 02000000:00000010:3.0:1784949054.900124:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000400:00000200:0.0:1784949054.900129:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:0.0:1784949054.900136:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000100:00000001:3.0:1784949054.900137:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900149:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1784949054.900156:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1784949054.900162:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949054 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949054.900214:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949054 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949054.900221:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949054.900227:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900236:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1784949054.900243:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c8e6480] to pc [ptlrpcd_00_03+3] req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949054.900244:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.900253:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949054.900259:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949054.900273:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949054 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949054.900284:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.900298:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949054.900301:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900311:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949054.900311:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900319:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1784949054.900320:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:0.0:1784949054.900324:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1784949054.900328:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949054.900337:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949054 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949054.900339:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949054.900352:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949054.900355:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949054.900367:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949054.900368:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:3.0:1784949054.900369:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000031b67b62. 00000100:00000001:1.0:1784949054.900380:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1784949054.900384:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949054.900386:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949054.900393:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949054.900394:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1784949054.900399:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949054.900403:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784949054.900404:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949054.900412:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900421:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000010:3.0:1784949054.900429:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:0.0:1784949054.900430:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949054.900435:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949054.900444:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.900447:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949054.900450:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1784949054.900459:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1784949054.900461:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1784949054.900472:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949054.900489:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949054.900491:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00100000:1.0:1784949054.900525:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005c8e6480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289670784:0@lo:400: 00000100:00000001:3.0:1784949054.900530:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949054.900543:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900545:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1784949054.900545:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949054.900565:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949054.900573:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949054.900574:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949054 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949054.900575:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000031b67b62] to pc [ptlrpcd_00_00+0] req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949054.900591:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1784949054.900659:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289669504:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784949054.900668:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949054.900670:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900673:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949054.900681:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949054.900683:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11331) 00000100:00000040:0.0:1784949054.900684:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000046cb422a x1871651289669504/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949054 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949054.900690:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949054.900697:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:2.0:1784949054.900712:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784949054.900714:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784949054.900720:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949054.900723:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1784949054.900728:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949054.900729:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1784949054.900729:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784949054.900743:0:89766:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1784949054.900744:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1784949054.900749:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949054.900754:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000c4a1e5eb. 00000100:00080000:3.0:1784949054.900761:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11301 02000000:00000001:0.0:1784949054.900773:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1784949054.900782:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1784949054.900783:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:3.0:1784949054.900783:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000020:00000040:0.0:1784949054.900793:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:2.0:1784949054.900794:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949054.900796:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11300 00000020:00000001:0.0:1784949054.900808:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:2.0:1784949054.900810:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:0.0:1784949054.900814:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000579609a4. 00000400:00000200:3.0:1784949054.900816:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 02000000:00000001:2.0:1784949054.900818:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949054.900828:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949054.900829:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949054.900830:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949054.900834:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:0.0:1784949054.900837:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1784949054.900838:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949054.900845:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000046cb422a. 00000100:00000001:3.0:1784949054.900847:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1784949054.900848:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289670912:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949054.900859:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000010:1.0:1784949054.900861:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 00000100:00000040:3.0:1784949054.900862:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949054.900866:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949054.900867:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949054.900880:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.900882:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949054.900883:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:2.0:1784949054.900885:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949054.900894:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:1.0:1784949054.900895:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e572277. 02000000:00000001:2.0:1784949054.900896:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949054.900896:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949054.900948:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949054.900953:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1784949054.900954:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:0.0:1784949054.900956:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949054.900962:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949054.900964:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:2.0:1784949054.900965:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007a6587e5. 02000000:00000001:2.0:1784949054.900973:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949054.900974:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:3.0:1784949054.900981:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:2.0:1784949054.900983:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b5489cc4. 00000020:00000001:3.0:1784949054.900986:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:2.0:1784949054.900996:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000100:00000040:3.0:1784949054.900998:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949054.901006:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1784949054.901011:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670912, portal 10 00000100:00000040:3.0:1784949054.901015:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949054.901018:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949054.901024:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:2.0:1784949054.901025:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554959744 : -115544154591872 : ffff96e9c7b5ad80) 00000020:00000040:3.0:1784949054.901036:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000040:2.0:1784949054.901049:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1784949054.901049:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000010:1.0:1784949054.901055:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000100:00080000:3.0:1784949054.901070:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000200:1.0:1784949054.901075:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289670784, portal 25 00000100:00000001:3.0:1784949054.901087:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.901089:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949054.901097:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554963328 : -115544154588288 : ffff96e9c7b5bb80) 00000100:00000001:2.0:1784949054.901099:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949054.901100:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1784949054.901115:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949054.901118:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949054.901126:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949054.901132:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949054.901139:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000100:00000001:3.0:1784949054.901148:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:2.0:1784949054.901154:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289670912, offset 0 00000100:00000001:3.0:1784949054.901159:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.901177:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:3.0:1784949054.901178:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000400:00000200:2.0:1784949054.901189:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000040:3.0:1784949054.901191:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:1.0:1784949054.901197:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949054.901209:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 02000000:00000001:3.0:1784949054.901210:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1784949054.901219:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000001:3.0:1784949054.901222:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949054.901224:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289670784, offset 0 00000400:00000200:2.0:1784949054.901231:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784949054.901239:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949054.901247:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949054.901253:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949054.901260:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949054.901262:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:3.0:1784949054.901267:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007617a2e3. 00000100:00000001:3.0:1784949054.901278:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949054.901279:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949054.901290:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00100000:3.0:1784949054.901295:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949054.901303:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3566:3566:241:4294967295] 192.168.202.137@tcp LPNI seq info [2291:2291:8:4294967295] 00000100:00000001:3.0:1784949054.901327:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949054.901333:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949054.901346:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_rcv+-1] req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949054.901348:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949054.901375:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949054.901389:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:2.0:1784949054.901390:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fb9675a2. 00000100:00000001:3.0:1784949054.901390:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949054.901417:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11331) 00000800:00000200:2.0:1784949054.901427:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949054.901430:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a480 00000400:00000200:1.0:1784949054.901445:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x82a9 [2] + 0 00000400:00000010:1.0:1784949054.901457:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002fbcdf42. 00000800:00000200:2.0:1784949054.901457:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784949054.901471:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fb9675a2 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1784949054.901472:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000049294c87 00000400:00000200:1.0:1784949054.901488:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949054.901509:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949054.901516:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000049294c87 00000400:00000010:1.0:1784949054.901518:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000049294c87. 00000100:00000001:1.0:1784949054.901523:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949054.901527:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784949054.901531:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000974405de x1871651289670784 msgsize 224 00000100:00000001:2.0:1784949054.901537:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.901544:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949054.901551:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949054.901551:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2145 usecs (3 jiffies) 00000100:00000001:0.0:1784949054.901555:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784949054.901558:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949054.901565:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949054.901569:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305994244 00000100:00000040:0.0:1784949054.901579:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949054.901582:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000010:2.0:1784949054.901583:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e2039f57. 00000400:00000200:1.0:1784949054.901598:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000100:00000001:2.0:1784949054.901604:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000400:00000010:1.0:1784949054.901605:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000100:00000001:0.0:1784949054.901606:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949054.901615:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949054.901624:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949054.901624:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949054.901629:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305994244 02000000:00000001:0.0:1784949054.901638:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1784949054.901643:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784949054.901649:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a3d0cc2f. 02000000:00000001:0.0:1784949054.901651:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949054.901664:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00100000:0.0:1784949054.901673:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289671040:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949054.901682:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305994244 00000100:00000001:0.0:1784949054.901699:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1784949054.901703:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000445487c0. 00000100:00000001:0.0:1784949054.901710:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.901713:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949054.901713:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949054.901721:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949054.901727:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305994244 00000100:00000040:1.0:1784949054.901729:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949054.901734:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949054.901746:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008a834f68. 00000100:00000001:0.0:1784949054.901751:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949054.901761:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949054.901772:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949054.901776:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949054.901782:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949054.901787:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949054.901792:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.901794:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949054.901795:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949054.901801:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.901810:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949054.901811:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949054.901813:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949054.901816:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949054.901819:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.901827:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305994283 00000100:00000001:2.0:1784949054.901828:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949054.901830:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949054.901830:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fb9675a2. 00000100:00000001:2.0:1784949054.901839:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:3.0:1784949054.901851:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949054.901852:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949054.901855:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4305994244 00000100:00000001:2.0:1784949054.901862:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949054.901875:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949054.901876:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949054.901877:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305994283 00000100:00000001:2.0:1784949054.901888:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.901892:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4305994244 00000400:00000200:3.0:1784949054.901899:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000100:00000001:1.0:1784949054.901929:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305994284 00000100:00000001:1.0:1784949054.901943:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4305994244 00000100:00000001:0.0:1784949054.901944:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949054.901947:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305994284 00000100:00100000:0.0:1784949054.901949:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670784 02000000:00000001:0.0:1784949054.901956:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949054.901957:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4305994244 00000400:00000010:3.0:1784949054.901960:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000100:00000001:0.0:1784949054.901966:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949054.901971:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949054.901974:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949054.901977:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949054.901979:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949054.901983:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949054.901987:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949054.901994:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949054.901997:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670784 00000100:00000200:3.0:1784949054.901998:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949054.902001:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949054.902004:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000001:2.0:1784949054.902008:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949054.902016:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000001:2.0:1784949054.902019:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949054.902021:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784949054.902029:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949054.902032:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000001:2.0:1784949054.902034:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949054.902040:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949054.902049:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784949054.902053:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949054.902055:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949054.902059:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949054.902066:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949054.902068:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:2.0:1784949054.902070:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:3.0:1784949054.902076:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949054.902086:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.902088:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949054.902104:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:3.0:1784949054.902108:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949054.902114:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a500 00000100:00000001:0.0:1784949054.902116:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949054.902118:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:0.0:1784949054.902129:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000400:00000200:2.0:1784949054.902142:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 167024 00000020:00000010:0.0:1784949054.902143:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001879354a. 00000020:00000010:0.0:1784949054.902156:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007194bf82. 00000800:00000001:2.0:1784949054.902160:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:0.0:1784949054.902165:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000800:00000001:2.0:1784949054.902174:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949054.902179:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:2.0:1784949054.902181:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949054.902189:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:2.0:1784949054.902196:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949054.902197:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949054.902205:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949054.902211:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949054.902216:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949054.902217:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949054.902223:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000128113ce. 00000100:00000040:2.0:1784949054.902230:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000128113ce x1871651289670912 msgsize 224 00000100:00100000:2.0:1784949054.902235:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949054.902242:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949054.902259:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949054.902261:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949054.902266:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949054.902275:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 765 00000800:00000001:2.0:1784949054.902276:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949054.902281:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949054.902287:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180543568 : -115539529008048 : ffff96eadb6a7c50) 00000800:00000001:2.0:1784949054.902288:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949054.902304:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000974405de x1871651289670784/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:715/0 lens 224/0 e 0 to 0 dl 1784949065 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949054.902319:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949054.902322:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000010:0.0:1784949054.902325:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00100000:3.0:1784949054.902328:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289670912 00000100:00000001:0.0:1784949054.902331:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949054.902335:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000010:0.0:1784949054.902338:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000100:00000001:3.0:1784949054.902341:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949054.902346:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949054.902351:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000974405de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289670784:12345-0@lo:400: 02000000:00000001:3.0:1784949054.902355:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949054.902360:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670784 00000100:00100000:3.0:1784949054.902365:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289670912 00000020:00000001:0.0:1784949054.902367:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949054.902374:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:0.0:1784949054.902376:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000040:3.0:1784949054.902380:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949054.902382:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949054.902388:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:0.0:1784949054.902389:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949054.902395:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000040:3.0:1784949054.902396:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949054.902402:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949054.902403:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949054.902407:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949054.902410:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949054.902413:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949054.902417:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:0.0:1784949054.902418:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949054.902426:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949054.902429:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000020:00000001:0.0:1784949054.902430:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949054.902436:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949054.902438:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949054.902442:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:3.0:1784949054.902444:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 02000000:00000010:0.0:1784949054.902451:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000079de2679. 02000000:00000001:0.0:1784949054.902459:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949054.902459:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000053ab16eb. 00000100:00000001:0.0:1784949054.902465:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949054.902470:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bac091eb. 00000020:00000001:0.0:1784949054.902471:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949054.902482:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670784 00000020:00000010:3.0:1784949054.902483:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00010000:00000001:0.0:1784949054.902486:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949054.902516:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000974405de x1871651289670784/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:715/0 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949054.902524:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00010000:00000001:0.0:1784949054.902528:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949054.902531:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00010000:00000001:0.0:1784949054.902533:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949054.902537:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00001000:0.0:1784949054.902540:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=76 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949054.902543:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.902548:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949054.902554:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949054.902555:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949054.902558:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949054.902562:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:0.0:1784949054.902564:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784949054.902567:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949054.902569:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.902577:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949054.902580:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949054.902586:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000100:00000001:3.0:1784949054.902588:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949054.902591:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289670784, offset 224 00000400:00000200:0.0:1784949054.902604:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949054.902610:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949054.902616:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:0.0:1784949054.902624:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:3.0:1784949054.902634:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 741 00000100:00000040:3.0:1784949054.902640:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949054.902644:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562011392 : -115544147540224 : ffff96e9c8214700) 00000400:00000200:0.0:1784949054.902652:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a480 00000100:00000040:3.0:1784949054.902661:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000128113ce x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:715/0 lens 224/0 e 0 to 0 dl 1784949065 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949054.902671:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8cc1 [1] + 224 00000400:00000200:0.0:1784949054.902690:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949054.902693:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949054.902700:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:3.0:1784949054.902701:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:0.0:1784949054.902707:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949054.902709:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949054.902715:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000100:00000200:0.0:1784949054.902719:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949054.902730:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000128113ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289670912:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949054.902747:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289670912 00000020:00000001:3.0:1784949054.902755:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000040:0.0:1784949054.902762:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949054.902763:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949054.902770:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949054.902779:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949054.902785:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949054.902791:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949054.902794:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949054.902796:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949054.902798:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949054.902801:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.902802:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1784949054.902804:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949054.902812:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:0.0:1784949054.902815:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 02000000:00000001:3.0:1784949054.902815:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000010:0.0:1784949054.902823:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 02000000:00000010:3.0:1784949054.902824:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000056522212. 00000100:00000001:0.0:1784949054.902831:0:342756:0:(events.c:386:reply_out_callback()) Process entered 02000000:00000001:3.0:1784949054.902832:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.902838:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949054.902839:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949054.902846:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.902848:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949054.902851:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289670912 00000100:00000040:0.0:1784949054.902856:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:3.0:1784949054.902860:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000001:0.0:1784949054.902864:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949054.902869:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784949054.902873:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000128113ce x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:715/0 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949054.902877:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949054.902881:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949054.902887:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949054.902889:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000974405de x1871651289670784/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:715/0 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949054.902899:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949054.902938:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949054.902942:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e572277. 00000400:00000200:1.0:1784949054.902955:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00010000:00000001:3.0:1784949054.902957:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000010:1.0:1784949054.902966:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00010000:00000001:3.0:1784949054.902968:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.902976:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784949054.902981:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000974405de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289670784:12345-0@lo:400: Request processed in 653us (1440us total) trans 0 rc 0/0 00000100:00001000:3.0:1784949054.902983:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=71 v=5 (1 1 1 1) 00000100:00000200:1.0:1784949054.902992:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949054.903007:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949054.903027:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00100000:0.0:1784949054.903036:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 765 00000100:00000001:3.0:1784949054.903043:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000040:0.0:1784949054.903054:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:1.0:1784949054.903055:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949054.903055:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949054.903063:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.903070:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949054.903079:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949054.903081:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:0.0:1784949054.903093:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007194bf82. 00000100:00000001:1.0:1784949054.903100:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784949054.903102:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949054.903109:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949054.903110:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000010:0.0:1784949054.903115:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001879354a. 00000100:00000001:1.0:1784949054.903126:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000040:0.0:1784949054.903129:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000010:3.0:1784949054.903130:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003dcd3ddc. 02000000:00000001:1.0:1784949054.903132:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:0.0:1784949054.903138:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949054.903139:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:3.0:1784949054.903142:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289670912, offset 224 02000000:00000010:0.0:1784949054.903153:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000079de2679. 00000100:00000001:1.0:1784949054.903154:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949054.903164:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949054.903171:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949054.903172:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949054.903180:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784949054.903183:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=76 v=5 (5 5 5 5) 00000100:00000200:0.0:1784949054.903190:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000200:3.0:1784949054.903191:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00001000:1.0:1784949054.903196:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=76 v=5 (1 1 1 1) 00000400:00000010:0.0:1784949054.903204:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d30ef19f. 00000100:00000001:1.0:1784949054.903213:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949054.903218:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949054.903220:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3567:3567:241:4294967295] 192.168.202.137@tcp LPNI seq info [2292:2292:8:4294967295] 00000400:00000010:0.0:1784949054.903221:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000be708ecf. 00010000:00000001:1.0:1784949054.903223:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949054.903237:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.903244:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949054.903257:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949054.903264:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949054.903297:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949054.903304:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949054.903311:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000010:3.0:1784949054.903311:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000100:00000001:1.0:1784949054.903320:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949054.903327:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:1.0:1784949054.903330:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949054.903352:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00100000:1.0:1784949054.903357:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005c8e6480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289670784:0@lo:400: 00000800:00000200:3.0:1784949054.903364:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949054.903369:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949054.903373:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949054.903375:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005c8e6480 x1871651289670784/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949054.903381:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949054.903383:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949054.903388:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 00000100:00000001:3.0:1784949054.903388:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949054.903403:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949054.903403:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000020:00000001:1.0:1784949054.903411:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00010000:00000001:3.0:1784949054.903412:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949054.903421:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949054.903423:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949054.903439:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:3.0:1784949054.903440:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 02000000:00000010:1.0:1784949054.903452:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 00000100:00000040:3.0:1784949054.903461:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000128113ce x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:715/0 lens 224/224 e 0 to 0 dl 1784949065 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949054.903463:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949054.903471:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949054.903478:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005c8e6480. 00000100:00000001:1.0:1784949054.903511:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949054.903514:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949054.903516:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949054.903520:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000128113ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289670912:12345-192.168.202.137@tcp:400: Request processed in 801us (1262us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949054.903525:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949054.903528:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949054.903538:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949054.903545:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949054.903545:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 741 00000100:00000040:3.0:1784949054.903558:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949054.903563:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949054.903570:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949054.903575:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bac091eb. 00000020:00000010:3.0:1784949054.903582:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949054.903588:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000053ab16eb. 00000020:00000040:3.0:1784949054.903594:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000800:00000200:0.0:1784949054.903596:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949054.903598:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784949054.903605:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:0.0:1784949054.903615:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949054.903624:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949054.903631:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003dcd3ddc 00000400:00000010:0.0:1784949054.903635:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003dcd3ddc. 00000100:00000001:0.0:1784949054.903640:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949054.903642:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949054.903653:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949054.903669:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949054.903674:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949054.903677:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949054.903687:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949054.903701:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a500 00000400:00000200:2.0:1784949054.903709:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8cbd [1] + 224 00000800:00000001:2.0:1784949054.903716:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949054.903726:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949054.903729:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949054.903734:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949054.903739:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949054.903745:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949054.903758:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949054.903788:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949054.903795:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949054.903799:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.903839:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949054.903844:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949054.903850:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949054.903852:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949054.903854:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b5489cc4. 00000400:00000200:0.0:1784949054.903861:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000010:0.0:1784949054.903863:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000100:00000001:0.0:1784949054.903866:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949054.903872:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949054.903883:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949054.903892:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949054.903894:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.903897:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949054.903900:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949054.903902:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949054.903904:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949054.903936:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949054.903943:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=71 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949054.903947:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=71 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949054.903951:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949054.903953:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949054.903956:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949054.903961:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949054.903970:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.903974:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949054.903975:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949054.903978:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949054.903979:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949054.903983:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949054.903991:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949054.903993:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949054.903995:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949054.903999:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949054.904016:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289670912:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949054.904022:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.904025:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949054.904027:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000031b67b62 x1871651289670912/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949070 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949054.904035:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949054.904037:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949054.904039:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007a6587e5. 02000000:00000001:0.0:1784949054.904046:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949054.904048:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949054.904050:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949054.904053:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949054.904055:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:0.0:1784949054.904059:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949054.904061:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949054.904062:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000031b67b62. 00000100:00000001:0.0:1784949054.904066:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949054.904068:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.904070:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949054.904075:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949054.904077:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949054.904079:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949054.904082:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949054.943998:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305994283, at 4305994284 00000100:00000010:1.0:1784949054.944012:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a3d0cc2f. 00000100:00000040:0.0:1784949054.944041:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305994283, at 4305994284 00000100:00000010:0.0:1784949054.944056:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00000100:00000040:3.0:1784949054.945028:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305994284, at 4305994285 00000100:00000040:2.0:1784949054.945040:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305994284, at 4305994285 00000100:00000010:3.0:1784949054.945044:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008a834f68. 00000100:00000010:2.0:1784949054.945057:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000445487c0. 00010000:00000001:2.0:1784949055.732059:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949055.732069:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949055.732076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949055.732079:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949055.732187:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949055.732196:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949055.732200:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949055.924044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949055.924054:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949055.924070:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949054/real 0] req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949055 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949055.924089:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.924092:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949055.924095:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949055.924100:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949055 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949055.924110:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.924121:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949055.924123:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.924126:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949055.924129:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949055.924132:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949055.924136:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949055 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949055.924145:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949055.924148:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.924150:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949055.924151:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.924154:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949055.924157:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949055.924161:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949055.924191:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784949055.924195:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949055.924200:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949055 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949055.924210:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289671040:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949055.924215:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.924218:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949055.924221:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289671040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949055 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949055.924230:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784949055.924233:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949055.924236:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784949055.924239:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949055.924245:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000007617a2e3. 02000000:00000001:0.0:1784949055.924253:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949055.924255:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949055.924258:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000100:00000001:0.0:1784949055.924263:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949055.924265:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.924268:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.924272:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949055.924274:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.924277:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949055.924279:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949055.924295:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949055.924308:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949055.924319:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000040:3.0:1784949055.924326:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949055.924332:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949055.924335:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949055.924342:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949055.924350:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949055.924358:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949055.924366:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949055.924381:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949055.924383:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949055.924390:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_01+1] req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949055.924428:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949055.924431:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949055.924435:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784949055.924438:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784949055.924440:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784949055.924445:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784949055.924450:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11301 00000400:00000200:3.0:1784949055.924463:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:3.0:1784949055.924468:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11326 00000100:00000001:0.0:1784949055.924474:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949055.924474:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:3.0:1784949055.924482:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1784949055.924484:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000040:3.0:1784949055.924511:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949055.924512:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.924523:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949055.924525:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784949055.924531:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949055.924532:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.924546:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784949055.924547:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949055.924555:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949055.924556:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1784949055.924563:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:0.0:1784949055.924564:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949055.924571:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949055.924572:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784949055.924578:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949055.924587:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:1.0:1784949055.924590:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949055.924608:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:0.0:1784949055.924613:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1784949055.924616:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:0.0:1784949055.924618:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949055.924628:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1784949055.924629:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000001:0.0:1784949055.924635:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949055.924636:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949055.924645:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 02000000:00000001:0.0:1784949055.924646:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.924650:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:0.0:1784949055.924656:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289671168:0@lo:400: 00000020:00000040:3.0:1784949055.924659:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:0.0:1784949055.924670:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1784949055.924672:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00080000:3.0:1784949055.924678:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 02000000:00000001:0.0:1784949055.924679:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949055.924684:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949055.924685:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949055.924691:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1784949055.924693:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949055.924702:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001879354a. 02000000:00000001:3.0:1784949055.924703:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949055.924708:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949055.924711:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.924716:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:0.0:1784949055.924718:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000da8fa306. 00000100:00000001:3.0:1784949055.924720:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949055.924732:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001445c4ff. 00000400:00000010:0.0:1784949055.924734:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000020:00000040:3.0:1784949055.924737:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784949055.924746:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000100:00000200:0.0:1784949055.924747:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289671168, portal 25 02000000:00000001:3.0:1784949055.924749:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.924759:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949055.924760:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949055.924768:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 02000000:00000001:3.0:1784949055.924768:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949055.924774:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949055.924781:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784949055.924785:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007617a2e3. 00000100:00000001:3.0:1784949055.924796:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.924801:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1784949055.924810:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784949055.924811:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949055.924818:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000100:00000200:0.0:1784949055.924826:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289671168, offset 0 00000100:00000001:3.0:1784949055.924840:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1784949055.924847:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949055.924848:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949055.924857:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001445c4ff] to pc [ptlrpcd_rcv+-1] req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949055.924870:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784949055.924884:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a600 00000100:00000001:3.0:1784949055.924888:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949055.924897:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x82c5 [2] + 0 00000100:00000040:3.0:1784949055.924898:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949055.924954:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949055.924956:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784949055.924957:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000100:00000010:3.0:1784949055.924961:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000100:00000001:1.0:1784949055.924971:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1784949055.924973:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:0.0:1784949055.924976:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000a1933a2b 00000100:00000001:1.0:1784949055.924977:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949055.924979:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949055.924991:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949055.924995:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949055.924997:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784949055.925010:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784949055.925010:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000400:00000200:0.0:1784949055.925015:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949055.925030:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949055.925037:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a1933a2b 00000400:00000010:0.0:1784949055.925051:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000a1933a2b. 00000100:00000040:3.0:1784949055.925058:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949055.925072:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949055.925080:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949055.925100:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000475c8050 x1871651289671168 msgsize 224 00000100:00000001:1.0:1784949055.925103:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949055.925112:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949055.925134:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:0.0:1784949055.925136:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784949055.925139:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949055.925152:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949055.925153:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 02000000:00000001:1.0:1784949055.925154:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949055.925170:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006d868603] to pc [ptlrpcd_00_02+2] req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949055.925172:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.925183:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949055.925200:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000400:00000010:0.0:1784949055.925207:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000100:00000001:0.0:1784949055.925213:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949055.925221:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949055.925233:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000001445c4ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289671296:192.168.202.137@tcp:8: 00000100:00000001:3.0:1784949055.925240:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.925248:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949055.925252:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949055.925264:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949055.925265:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949055.925267:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289671168 00000100:00000040:3.0:1784949055.925270:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11332) 02000000:00000001:2.0:1784949055.925282:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:1.0:1784949055.925284:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949055.925301:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949055.925303:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949055.925304:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.925307:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949055.925317:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.925323:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784949055.925330:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949055.925331:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949055.925332:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1784949055.925333:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949055.925350:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289671168 00000100:00000001:0.0:1784949055.925354:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949055.925357:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000008905542c. 00000020:00000001:2.0:1784949055.925357:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:3.0:1784949055.925357:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784949055.925368:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 02000000:00000001:1.0:1784949055.925381:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949055.925382:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949055.925384:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4305995307 00000020:00000040:2.0:1784949055.925394:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:0.0:1784949055.925409:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000400:00000010:1.0:1784949055.925409:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e572277. 00000020:00000001:2.0:1784949055.925410:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949055.925412:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949055.925422:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949055.925437:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:3.0:1784949055.925439:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949055.925441:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000010:1.0:1784949055.925442:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000100:00000001:2.0:1784949055.925445:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949055.925460:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000200:1.0:1784949055.925465:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289671296, portal 4 00000100:00000001:0.0:1784949055.925466:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4305995307 00000100:00000001:2.0:1784949055.925470:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949055.925475:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1784949055.925477:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1784949055.925482:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000020:00000010:2.0:1784949055.925484:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00000001:1.0:1784949055.925486:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554962432 : -115544154589184 : ffff96e9c7b5b800) 02000000:00000001:3.0:1784949055.925515:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.925521:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949055.925536:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4305995307 00000100:00100000:3.0:1784949055.925537:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289671424:192.168.202.137@tcp:400: 00000020:00000010:2.0:1784949055.925542:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000100:00000040:1.0:1784949055.925543:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784949055.925553:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:3.0:1784949055.925572:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949055.925573:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:2.0:1784949055.925576:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004f0879c9. 00000100:00000001:1.0:1784949055.925587:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949055.925588:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949055.925590:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4305995307 02000000:00000001:3.0:1784949055.925594:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949055.925595:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000010:0.0:1784949055.925610:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000040:1.0:1784949055.925610:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949055.925612:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784949055.925622:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 02000000:00000010:3.0:1784949055.925634:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000053ab16eb. 00000400:00000010:1.0:1784949055.925636:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000100:00000001:0.0:1784949055.925637:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949055.925638:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949055.925648:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:1.0:1784949055.925655:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289671296, offset 0 02000000:00000001:3.0:1784949055.925656:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.925659:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949055.925659:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949055.925681:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000100:00000001:0.0:1784949055.925684:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949055.925689:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.925698:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949055.925704:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949055.925713:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949055.925722:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000100:00000001:2.0:1784949055.925727:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949055.925736:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289671424, portal 10 00000400:00000200:1.0:1784949055.925740:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949055.925748:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949055.925749:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949055.925753:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554951680 : -115544154599936 : ffff96e9c7b58e00) 00000400:00000200:1.0:1784949055.925755:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784949055.925756:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949055.925770:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949055.925779:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949055.925781:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:3.0:1784949055.925787:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949055.925798:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 766 00000400:00000200:1.0:1784949055.925811:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1784949055.925816:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949055.925829:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180529232 : -115539529022384 : ffff96eadb6a4450) 00000400:00000200:1.0:1784949055.925833:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949055.925849:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949055.925857:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000475c8050 x1871651289671168/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:716/0 lens 224/0 e 0 to 0 dl 1784949066 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949055.925864:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3568:3568:241:4294967295] 192.168.202.137@tcp LPNI seq info [2293:2293:8:4294967295] 00000100:00000040:3.0:1784949055.925877:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949055.925893:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000100:00000200:3.0:1784949055.925958:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289671424, offset 0 00000100:00000001:2.0:1784949055.925984:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949055.925993:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000010:2.0:1784949055.925997:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000400:00000200:1.0:1784949055.926013:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784949055.926014:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949055.926035:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000800:00000200:1.0:1784949055.926047:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949055.926053:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00100000:2.0:1784949055.926063:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000475c8050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289671168:12345-0@lo:400: 00000800:00000010:1.0:1784949055.926073:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000af953512. 00000400:00000200:3.0:1784949055.926079:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:2.0:1784949055.926082:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289671168 00000020:00000001:2.0:1784949055.926095:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:3.0:1784949055.926100:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:2.0:1784949055.926104:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000200:1.0:1784949055.926113:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:2.0:1784949055.926116:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949055.926125:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1784949055.926132:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:3.0:1784949055.926137:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:2.0:1784949055.926139:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:1.0:1784949055.926151:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:3.0:1784949055.926160:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3569:3569:240:4294967295] 192.168.202.137@tcp LPNI seq info [2294:2294:7:4294967295] 00000020:00000001:2.0:1784949055.926161:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000200:1.0:1784949055.926167:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000af953512 type 1, nob 616 niov 1 nkiov 1 00000020:00000001:2.0:1784949055.926168:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949055.926189:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949055.926197:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949055.926203:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949055.926204:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:2.0:1784949055.926210:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949055.926217:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949055.926220:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949055.926224:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:1.0:1784949055.926231:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000800:00000200:3.0:1784949055.926233:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000010:2.0:1784949055.926234:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a6fbdd8b. 00000100:00000001:1.0:1784949055.926241:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305995308 02000000:00000001:2.0:1784949055.926244:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949055.926257:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000100:00000001:2.0:1784949055.926258:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949055.926268:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949055.926269:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4305995307 00010000:00000040:2.0:1784949055.926283:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289671168 00000800:00000200:3.0:1784949055.926286:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949055.926287:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305995308 00010000:00000001:2.0:1784949055.926290:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949055.926310:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4305995307 00010000:00000200:2.0:1784949055.926313:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000475c8050 x1871651289671168/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:716/0 lens 224/224 e 0 to 0 dl 1784949066 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949055.926322:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305995308 00000800:00000200:3.0:1784949055.926329:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949055.926340:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4305995307 00000800:00000200:3.0:1784949055.926352:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949055.926359:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305995308 00010000:00000001:2.0:1784949055.926370:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949055.926377:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949055.926380:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4305995307 00000100:00001000:2.0:1784949055.926384:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=77 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949055.926387:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949055.926393:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949055.926407:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949055.926407:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949055.926412:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2147 usecs (3 jiffies) 00000800:00000200:0.0:1784949055.926426:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949055.926429:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949055.926432:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949055.926432:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4305995269 00000800:00000010:0.0:1784949055.926436:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000af953512. 00000100:00000001:1.0:1784949055.926458:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.926460:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4305995308 00000100:00000001:2.0:1784949055.926461:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:0.0:1784949055.926463:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784949055.926475:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784949055.926477:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4305995269 00000100:00000001:1.0:1784949055.926478:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1784949055.926481:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949055.926519:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1784949055.926520:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949055.926520:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4305995308 00000100:00000001:2.0:1784949055.926523:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949055.926541:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4305995269 00000100:00000001:1.0:1784949055.926543:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949055.926545:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:0.0:1784949055.926546:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000100:00000001:1.0:1784949055.926551:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949055.926556:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4305995308 00000400:00000010:0.0:1784949055.926560:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000100:00000001:1.0:1784949055.926563:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949055.926565:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000001:3.0:1784949055.926574:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4305995269 00000100:00000001:0.0:1784949055.926577:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949055.926577:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.926585:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4305995308 00000100:00000200:2.0:1784949055.926587:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289671168, offset 224 00000100:00000001:3.0:1784949055.926598:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784949055.926599:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949055.926609:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949055.926611:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949055.926624:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.926631:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949055.926632:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949055.926640:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:1.0:1784949055.926645:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949055.926649:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949055.926651:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949080 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949055.926660:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784949055.926672:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949055.926674:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a600 00000800:00000001:1.0:1784949055.926682:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.926685:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.926685:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949055.926692:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8cd9 [1] + 224 00000100:00000001:3.0:1784949055.926697:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.926711:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784949055.926713:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949055.926714:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949055.926717:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949055.926737:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949055.926741:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949055.926754:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1784949055.926765:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784949055.926771:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949055.926796:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949055.926797:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643a680 00000400:00000100:1.0:1784949055.926832:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289671296 offset 0 length 520: 4 00000800:00000200:0.0:1784949055.926847:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1784949055.926866:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000000f594655 is in inconsistent state, don't perform health checking (-2, 0) 00000800:00000010:0.0:1784949055.926867:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000800:00000001:3.0:1784949055.926872:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949055.926877:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784949055.926882:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:0.0:1784949055.926887:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949055.926897:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000800:00000001:3.0:1784949055.926898:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949055.926900:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949055.926941:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949055.926946:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949055.926964:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:2.0:1784949055.926983:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000800:00000001:1.0:1784949055.926986:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949055.926997:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a700 00000100:00000001:2.0:1784949055.926998:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:0.0:1784949055.927001:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000100:00000001:2.0:1784949055.927004:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000010:0.0:1784949055.927009:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000800:00000001:1.0:1784949055.927017:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949055.927018:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.927020:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949055.927040:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949055.927040:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 167248 00000800:00000001:1.0:1784949055.927043:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949055.927050:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000800:00000001:1.0:1784949055.927057:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949055.927064:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949055.927076:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949055.927082:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949055.927084:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949055.927085:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949055.927099:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000475c8050 x1871651289671168/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:716/0 lens 224/224 e 0 to 0 dl 1784949066 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949055.927103:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949055.927105:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949055.927117:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949055.927118:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949055.927128:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949055.927144:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:3.0:1784949055.927153:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000001:1.0:1784949055.927156:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949055.927169:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000da8fa306. 00000400:00000200:1.0:1784949055.927189:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000100:00000001:0.0:1784949055.927191:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949055.927191:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000475c8050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289671168:12345-0@lo:400: Request processed in 1202us (2083us total) trans 0 rc 0/0 00000400:00000010:1.0:1784949055.927205:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000100:00000001:0.0:1784949055.927206:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949055.927227:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949055.927230:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 766 00000100:00000001:3.0:1784949055.927231:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949055.927240:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:1.0:1784949055.927244:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949055.927245:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000010:3.0:1784949055.927256:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001f1afd3c. 00000100:00000001:2.0:1784949055.927264:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949055.927276:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:3.0:1784949055.927279:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000001f1afd3c x1871651289671424 msgsize 224 00000020:00000010:2.0:1784949055.927289:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004f0879c9. 00000020:00000010:2.0:1784949055.927306:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000200:1.0:1784949055.927315:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949055.927319:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 00000100:00100000:3.0:1784949055.927339:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000040:2.0:1784949055.927340:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949055.927365:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1784949055.927365:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949055.927378:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949055.927380:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784949055.927381:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a6fbdd8b. 00000800:00000001:3.0:1784949055.927396:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949055.927400:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949055.927402:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000800:00000001:3.0:1784949055.927408:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949055.927411:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949055.927412:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949055.927427:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784949055.927430:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949055.927433:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949055.927439:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a31fc1e7. 02000000:00000001:1.0:1784949055.927441:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.927444:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949055.927461:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289671424 00000400:00000010:2.0:1784949055.927463:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fea29ebd. 00000100:00001000:1.0:1784949055.927464:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=77 v=5 (5 5 5 5) 02000000:00000001:0.0:1784949055.927469:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949055.927476:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949055.927509:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949055.927511:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=77 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949055.927521:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949055.927523:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949055.927532:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949055.927533:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289671424 00000020:00000001:0.0:1784949055.927549:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00010000:00000001:1.0:1784949055.927551:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949055.927565:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949055.927566:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949055.927576:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949055.927578:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949055.927584:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:1.0:1784949055.927594:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949055.927596:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949055.927614:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949055.927627:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949055.927632:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949055.927636:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:1.0:1784949055.927640:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949055.927647:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949055.927648:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949055.927671:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000100:00000040:1.0:1784949055.927677:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949055.927688:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00100000:1.0:1784949055.927697:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289671168:0@lo:400: 00000020:00000010:0.0:1784949055.927699:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007194bf82. 00000020:00000010:0.0:1784949055.927719:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949055.927719:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949055.927734:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949055.927745:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289671168/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949055.927748:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949055.927764:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949055.927771:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949055.927778:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784949055.927781:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949055.927784:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949055.927790:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001879354a. 00000100:00000001:0.0:1784949055.927792:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949055.927798:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:1.0:1784949055.927798:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1784949055.927807:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949055.927808:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949055.927816:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949055.927821:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949055.927825:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:1.0:1784949055.927831:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949055.927833:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1784949055.927834:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784949055.927836:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:1.0:1784949055.927846:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949055.927851:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949055.927856:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949055.927858:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949055.927868:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949055.927872:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949055.927880:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949055.927885:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 742 00000100:00000001:1.0:1784949055.927890:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949055.927892:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949055.927896:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.927898:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554948992 : -115544154602624 : ffff96e9c7b58380) 00000100:00000040:0.0:1784949055.927966:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001f1afd3c x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:716/0 lens 224/0 e 0 to 0 dl 1784949066 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949055.927979:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949055.927984:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:0.0:1784949055.927987:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949055.927990:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000100:00100000:0.0:1784949055.927996:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001f1afd3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289671424:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949055.928000:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289671424 00000020:00000001:0.0:1784949055.928003:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949055.928007:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949055.928009:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949055.928013:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949055.928016:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949055.928020:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949055.928023:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949055.928026:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949055.928029:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949055.928032:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949055.928035:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949055.928039:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949055.928042:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949055.928045:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000060c7f82e. 02000000:00000001:0.0:1784949055.928048:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.928051:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949055.928053:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949055.928056:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289671424 00010000:00000001:0.0:1784949055.928059:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949055.928063:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001f1afd3c x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:716/0 lens 224/224 e 0 to 0 dl 1784949066 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949055.928073:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949055.928076:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949055.928079:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=72 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949055.928084:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949055.928087:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949055.928090:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949055.928093:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949055.928095:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949055.928098:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949055.928100:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949055.928104:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000100:00000200:0.0:1784949055.928108:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289671424, offset 224 00000400:00000200:0.0:1784949055.928113:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949055.928123:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949055.928129:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3570:3570:241:4294967295] 192.168.202.137@tcp LPNI seq info [2295:2295:8:4294967295] 00000400:00000200:0.0:1784949055.928141:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949055.928147:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949055.928151:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008746b1e6. 00000800:00000200:0.0:1784949055.928157:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949055.928164:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949055.928168:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008746b1e6 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949055.928186:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949055.928190:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949055.928193:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949055.928195:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949055.928197:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949055.928201:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001f1afd3c x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:716/0 lens 224/224 e 0 to 0 dl 1784949066 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949055.928213:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001f1afd3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289671424:12345-192.168.202.137@tcp:400: Request processed in 233us (933us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949055.928223:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 742 00000100:00000040:0.0:1784949055.928228:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949055.928233:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949055.928236:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949055.928240:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007194bf82. 00000020:00000010:0.0:1784949055.928245:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949055.928249:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 00000020:00000040:0.0:1784949055.928255:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949055.928258:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949055.928371:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949055.928379:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008746b1e6. 00000400:00000200:1.0:1784949055.928385:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949055.928399:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949055.928406:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000400:00000010:1.0:1784949055.928411:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000800:00000001:0.0:1784949055.928414:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949055.928417:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949055.928421:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949055.928431:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949055.928437:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949055.928444:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949055.928454:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949055.928466:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a700 00000400:00000200:0.0:1784949055.928473:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8ce9 [1] + 224 00000800:00000001:0.0:1784949055.928480:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949055.928486:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949055.928513:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949055.928518:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949055.928524:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949055.928529:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949055.928566:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949055.928587:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949055.928595:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949055.928599:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.928641:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949055.928647:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949055.928654:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949055.928656:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949055.928659:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 00000400:00000200:3.0:1784949055.928666:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:3.0:1784949055.928669:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000100:00000001:3.0:1784949055.928673:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949055.928680:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949055.928692:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949055.928701:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949055.928704:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.928707:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949055.928710:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949055.928712:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949055.928714:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949055.928716:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949055.928723:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=72 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949055.928727:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=72 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949055.928730:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949055.928732:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949055.928735:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949055.928739:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949055.928747:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.928751:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949055.928753:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949055.928756:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949055.928758:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949055.928762:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949055.928770:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949055.928773:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949055.928775:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949055.928777:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949055.928785:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289671424:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949055.928790:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.928792:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949055.928795:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006d868603 x1871651289671424/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949071 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949055.928802:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949055.928804:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949055.928807:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000053ab16eb. 02000000:00000001:3.0:1784949055.928810:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949055.928812:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949055.928814:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949055.928816:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949055.928819:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:3.0:1784949055.928822:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949055.928824:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949055.928825:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006d868603. 00000100:00000001:3.0:1784949055.928842:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949055.928843:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.928845:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949055.928850:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949055.928852:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949055.928854:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949055.928856:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949055.969048:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4305995308, at 4305995309 00000100:00000010:2.0:1784949055.969062:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:3.0:1784949055.969092:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4305995308, at 4305995309 00000100:00000040:1.0:1784949055.969101:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4305995308, at 4305995309 00000100:00000040:0.0:1784949055.969103:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4305995308, at 4305995309 00000100:00000010:3.0:1784949055.969109:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:1.0:1784949055.969121:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000010:0.0:1784949055.969126:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00010000:00000001:2.0:1784949056.756066:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949056.756079:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949056.756093:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949056.756095:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949056.756170:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949056.756176:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949056.756181:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949057.780039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949057.780048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949057.780057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949057.780059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949057.780154:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949057.780162:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949057.780166:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949058.804030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949058.804037:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949058.804044:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949058.804046:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949058.804173:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949058.804180:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949058.804185:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949059.828032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949059.828041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949059.828051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949059.828054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949059.828172:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949059.828180:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949059.828185:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949060.852018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949060.852025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949060.852032:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949060.852033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949060.852166:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949060.852173:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949060.852177:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949061.044024:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949061.044042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949061.044046:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.044052:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949061.044056:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949061.044061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949061.044065:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949061.044089:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949061.044099:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949061.044108:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000002390dcc6. 00000020:00000040:2.0:1784949061.044116:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949061.044122:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949061.044125:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949061.044132:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949061.044142:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fb9675a2. 00000100:00000001:2.0:1784949061.044150:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949061.044161:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949061.044176:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949061.044179:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949061.044185:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002390dcc6] to pc [ptlrpcd_00_03+3] req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949061.044221:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949061.044224:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949061.044229:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949061.044232:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949061.044236:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949061.044239:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59951e9. 00000020:00000040:2.0:1784949061.044242:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949061.044244:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949061.044246:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949061.044248:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949061.044252:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f95b4924. 00000100:00000001:2.0:1784949061.044255:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949061.044262:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949061.044272:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949061.044275:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949061.044280:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f59951e9] to pc [ptlrpcd_00_00+0] req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949061.044303:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949061.044309:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949061.044314:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000040:2.0:1784949061.044318:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11338) 00000100:00000001:0.0:1784949061.044321:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949061.044326:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949061.044332:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949061.044343:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949061.044350:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.044353:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949061.044363:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949061.044365:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:3.0:1784949061.044374:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949061.044375:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949061.044377:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949061.044387:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949061.044399:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949061.044413:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949061.044423:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949061.044424:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949061.044438:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949061.044442:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949061.044484:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949061.044505:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949061.044505:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949061.044515:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949061.044539:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949061.044541:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949061.044549:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949061.044552:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949061.044556:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949061.044564:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949061.044566:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949061.044571:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949061.044580:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000002390dcc6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289671552:0@lo:400: 00000100:00100000:1.0:1784949061.044581:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f59951e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289671680:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949061.044604:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949061.044608:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949061.044613:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949061.044618:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949061.044622:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1784949061.044622:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949061.044634:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949061.044642:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 02000000:00000001:3.0:1784949061.044643:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1784949061.044655:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949061.044662:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000053ab16eb. 00000400:00000010:1.0:1784949061.044667:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e5b8264. 02000000:00000001:3.0:1784949061.044671:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949061.044680:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000010:3.0:1784949061.044687:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 00000100:00000200:1.0:1784949061.044693:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289671680, portal 10 00000100:00000001:1.0:1784949061.044701:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949061.044705:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562009600 : -115544147542016 : ffff96e9c8214000) 00000400:00000010:3.0:1784949061.044711:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000100:00000040:1.0:1784949061.044718:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949061.044728:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289671552, portal 25 00000100:00000001:1.0:1784949061.044731:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949061.044744:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1784949061.044745:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949061.044756:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562014080 : -115544147537536 : ffff96e9c8215180) 00000400:00000010:1.0:1784949061.044757:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000100:00000200:1.0:1784949061.044768:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289671680, offset 0 00000100:00000040:3.0:1784949061.044781:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949061.044785:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949061.044814:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949061.044821:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949061.044826:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000100:00000200:3.0:1784949061.044832:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289671552, offset 0 00000400:00000200:1.0:1784949061.044839:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949061.044849:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949061.044852:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949061.044862:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949061.044868:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949061.044873:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949061.044878:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3571:3571:241:4294967295] 192.168.202.137@tcp LPNI seq info [2296:2296:8:4294967295] 00000400:00000200:1.0:1784949061.044896:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949061.044906:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949061.044915:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a1bdc144. 00000400:00000200:3.0:1784949061.044951:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784949061.044972:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a780 00000800:00000200:1.0:1784949061.044983:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:3.0:1784949061.044989:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x82e1 [2] + 0 00000400:00000010:3.0:1784949061.045001:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000096c4b4ae. 00000400:00000200:3.0:1784949061.045010:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000059548475 00000800:00000200:1.0:1784949061.045015:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949061.045024:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a1bdc144 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1784949061.045026:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949061.045037:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949061.045044:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000059548475 00000400:00000010:3.0:1784949061.045048:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000059548475. 00000100:00000001:3.0:1784949061.045055:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949061.045060:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949061.045060:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949061.045083:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000a73b72ec x1871651289671552 msgsize 224 00000100:00000040:1.0:1784949061.045086:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2150 usecs (3 jiffies) 00000100:00000001:1.0:1784949061.045094:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306000388 00000100:00100000:3.0:1784949061.045110:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000010:1.0:1784949061.045112:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000040:3.0:1784949061.045115:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949061.045138:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949061.045151:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949061.045154:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306000388 00000100:00000010:1.0:1784949061.045162:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000400:00000200:3.0:1784949061.045165:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000100:00000001:1.0:1784949061.045169:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000400:00000010:3.0:1784949061.045172:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000100:00000001:1.0:1784949061.045186:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306000388 00000100:00000001:3.0:1784949061.045187:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:1.0:1784949061.045205:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000200:3.0:1784949061.045213:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949061.045214:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949061.045224:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306000388 00000100:00000010:1.0:1784949061.045236:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:1.0:1784949061.045245:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949061.045249:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949061.045257:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784949061.045261:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289671552 00000800:00000200:0.0:1784949061.045269:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949061.045270:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949061.045272:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949061.045275:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949061.045291:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000010:0.0:1784949061.045294:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a1bdc144. 00000100:00000001:1.0:1784949061.045296:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949061.045298:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949061.045301:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949061.045319:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949061.045321:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784949061.045322:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949061.045347:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289671552 00000100:00000001:3.0:1784949061.045352:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949061.045353:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949061.045365:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784949061.045368:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949061.045369:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000020:00000001:2.0:1784949061.045369:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:0.0:1784949061.045382:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000020:00000040:2.0:1784949061.045382:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949061.045389:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949061.045393:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000001:1.0:1784949061.045396:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949061.045398:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949061.045400:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306000427 00000020:00000040:2.0:1784949061.045404:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000800:00000001:1.0:1784949061.045412:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784949061.045422:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949061.045425:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949061.045428:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306000388 00000800:00000001:1.0:1784949061.045430:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949061.045487:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949061.045490:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306000427 00000100:00000001:3.0:1784949061.045508:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306000388 00000100:00000001:2.0:1784949061.045512:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949061.045515:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784949061.045519:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949061.045526:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306000427 00000020:00000010:2.0:1784949061.045533:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000040:0.0:1784949061.045536:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949061.045543:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306000388 00000100:00000001:2.0:1784949061.045547:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949061.045561:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306000427 00000400:00000200:1.0:1784949061.045569:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a800 00000020:00000010:2.0:1784949061.045581:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:3.0:1784949061.045582:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306000388 00000100:00000001:3.0:1784949061.045600:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949061.045607:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e5260462. 00000400:00000200:1.0:1784949061.045609:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 167472 00000100:00000001:0.0:1784949061.045613:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949061.045619:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949061.045631:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:2.0:1784949061.045633:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004f0879c9. 00000100:00000001:3.0:1784949061.045644:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949061.045652:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000800:00000001:1.0:1784949061.045653:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949061.045654:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949061.045688:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949061.045688:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:1.0:1784949061.045701:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949061.045705:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:1.0:1784949061.045715:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949061.045716:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949061.045723:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949061.045725:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949061.045734:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784949061.045741:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a612e3ff. 00000100:00000001:2.0:1784949061.045745:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949061.045748:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000a612e3ff x1871651289671680 msgsize 224 00000100:00100000:1.0:1784949061.045768:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784949061.045784:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.045791:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949061.045820:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:1.0:1784949061.045822:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949061.045825:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:1.0:1784949061.045831:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949061.045846:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 767 00000100:00000040:2.0:1784949061.045856:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949061.045861:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180532304 : -115539529019312 : ffff96eadb6a5050) 00000100:00000040:2.0:1784949061.045871:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a73b72ec x1871651289671552/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:722/0 lens 224/0 e 0 to 0 dl 1784949072 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949061.045885:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949061.045887:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:3.0:1784949061.045895:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289671680 00000020:00000010:2.0:1784949061.045896:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 02000000:00000001:3.0:1784949061.045903:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949061.045938:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949061.045947:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:3.0:1784949061.045948:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949061.045954:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949061.045961:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a73b72ec pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289671552:12345-0@lo:400: 02000000:00000001:3.0:1784949061.045964:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949061.045975:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289671552 00000100:00100000:3.0:1784949061.045977:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289671680 00000020:00000001:2.0:1784949061.045986:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949061.045986:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949061.045992:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949061.045996:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949061.045997:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:2.0:1784949061.046005:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949061.046006:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949061.046013:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949061.046014:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949061.046018:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949061.046021:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949061.046029:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949061.046030:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:2.0:1784949061.046040:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949061.046041:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:3.0:1784949061.046052:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1784949061.046054:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000010:3.0:1784949061.046057:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000020:00000001:2.0:1784949061.046063:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000010:3.0:1784949061.046068:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000fff5e130. 00000020:00000001:2.0:1784949061.046072:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949061.046077:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f94924e3. 00000020:00000001:2.0:1784949061.046081:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000010:3.0:1784949061.046085:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:2.0:1784949061.046089:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:3.0:1784949061.046098:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:2.0:1784949061.046099:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784949061.046107:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949061.046109:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000010:2.0:1784949061.046113:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a6fbdd8b. 00000100:00000001:3.0:1784949061.046113:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949061.046125:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949061.046126:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949061.046133:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949061.046136:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949061.046137:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949061.046150:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949061.046159:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289671552 00010000:00000001:2.0:1784949061.046165:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949061.046168:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:2.0:1784949061.046176:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a73b72ec x1871651289671552/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:722/0 lens 224/224 e 0 to 0 dl 1784949072 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949061.046184:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949061.046189:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000001:2.0:1784949061.046195:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949061.046201:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949061.046206:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 743 00000100:00001000:2.0:1784949061.046209:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=83 v=5 (1 1 1 1) 00000100:00000040:3.0:1784949061.046218:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949061.046221:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949061.046230:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949061.046230:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549158016 : -115544160393600 : ffff96e9c75d2680) 00000100:00000001:2.0:1784949061.046239:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784949061.046246:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949061.046249:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949061.046255:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a612e3ff x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:722/0 lens 224/0 e 0 to 0 dl 1784949072 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949061.046257:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949061.046266:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949061.046273:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000200:2.0:1784949061.046283:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289671552, offset 224 00000400:00000200:2.0:1784949061.046297:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949061.046302:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784949061.046311:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:3.0:1784949061.046312:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:3.0:1784949061.046319:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949061.046322:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000400:00000200:2.0:1784949061.046329:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a780 00000100:00100000:3.0:1784949061.046339:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a612e3ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289671680:12345-192.168.202.137@tcp:400: 00000400:00000200:2.0:1784949061.046342:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8d01 [1] + 224 00000400:00000200:2.0:1784949061.046358:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:3.0:1784949061.046360:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289671680 00000400:00000200:2.0:1784949061.046368:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:3.0:1784949061.046373:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949061.046376:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784949061.046383:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:2.0:1784949061.046384:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949061.046393:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949061.046407:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:2.0:1784949061.046411:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949061.046415:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949061.046429:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949061.046437:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949061.046443:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949061.046471:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1784949061.046474:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949061.046487:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949061.046491:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000100:00000001:0.0:1784949061.046492:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949061.046493:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000010:2.0:1784949061.046498:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:0.0:1784949061.046501:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949061.046505:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784949061.046506:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000001:0.0:1784949061.046513:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784949061.046515:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 02000000:00000001:3.0:1784949061.046516:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000001:0.0:1784949061.046522:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784949061.046526:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949061.046530:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f6dbc39d. 02000000:00000010:3.0:1784949061.046530:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e6df9f3d. 00000100:00000040:2.0:1784949061.046539:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 02000000:00000001:3.0:1784949061.046542:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949061.046545:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00010000:00000001:2.0:1784949061.046551:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949061.046556:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949061.046557:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000020:00000001:2.0:1784949061.046558:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949061.046569:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784949061.046570:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949061.046571:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:0.0:1784949061.046581:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:3.0:1784949061.046583:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289671680 00000100:00000040:2.0:1784949061.046590:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a73b72ec x1871651289671552/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:722/0 lens 224/224 e 0 to 0 dl 1784949072 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949061.046600:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949061.046617:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a612e3ff x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:722/0 lens 224/224 e 0 to 0 dl 1784949072 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949061.046619:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949061.046674:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a73b72ec pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289671552:12345-0@lo:400: Request processed in 777us (1579us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949061.046693:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000001:3.0:1784949061.046696:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949061.046701:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949061.046705:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949061.046712:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 767 00000100:00000001:0.0:1784949061.046713:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00001000:3.0:1784949061.046718:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=78 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949061.046721:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000040:2.0:1784949061.046722:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949061.046726:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949061.046733:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949061.046734:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949061.046737:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949061.046740:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:0.0:1784949061.046742:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949061.046750:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004f0879c9. 00000100:00000040:3.0:1784949061.046752:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00001000:0.0:1784949061.046753:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=83 v=5 (5 5 5 5) 00000020:00000010:2.0:1784949061.046762:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00001000:0.0:1784949061.046766:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=83 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949061.046768:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000010:2.0:1784949061.046770:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e5260462. 00000100:00000001:0.0:1784949061.046783:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000001:3.0:1784949061.046785:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000040:2.0:1784949061.046787:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784949061.046789:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949061.046793:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949061.046808:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000001:2.0:1784949061.046811:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:0.0:1784949061.046816:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949061.046818:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:2.0:1784949061.046819:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a6fbdd8b. 00000100:00000001:0.0:1784949061.046822:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949061.046829:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000040:3.0:1784949061.046835:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949061.046837:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949061.046838:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949061.046846:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000100:00000200:2.0:1784949061.046850:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000200:3.0:1784949061.046864:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289671680, offset 224 00000400:00000010:2.0:1784949061.046865:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001d3d7473. 00000100:00000001:0.0:1784949061.046866:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949061.046877:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:2.0:1784949061.046880:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fe8db63d. 00000100:00000001:0.0:1784949061.046881:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949061.046892:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949061.046894:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949061.046945:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000002390dcc6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289671552:0@lo:400: 00000100:00000001:0.0:1784949061.046953:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949061.046957:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949061.046963:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000002390dcc6 x1871651289671552/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949061.046978:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949061.046981:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949061.046987:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949061.046992:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000053ab16eb. 02000000:00000001:0.0:1784949061.047002:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1784949061.047003:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3572:3572:241:4294967295] 192.168.202.137@tcp LPNI seq info [2297:2297:8:4294967295] 00000020:00000001:0.0:1784949061.047008:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949061.047013:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949061.047021:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949061.047023:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fb9675a2. 02000000:00000001:0.0:1784949061.047030:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949061.047035:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949061.047039:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000002390dcc6. 00000100:00000001:0.0:1784949061.047048:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000400:00000200:3.0:1784949061.047050:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949061.047052:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949061.047060:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949061.047074:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949061.047078:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949061.047080:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949061.047083:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949061.047088:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949061.047097:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949061.047113:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949061.047128:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949061.047135:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949061.047165:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949061.047172:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949061.047177:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949061.047181:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949061.047186:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949061.047196:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a612e3ff x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:722/0 lens 224/224 e 0 to 0 dl 1784949072 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949061.047218:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a612e3ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289671680:12345-192.168.202.137@tcp:400: Request processed in 912us (1465us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949061.047235:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 743 00000100:00000040:3.0:1784949061.047243:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949061.047249:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949061.047252:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949061.047258:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f94924e3. 00000020:00000010:3.0:1784949061.047266:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000800:00000200:0.0:1784949061.047272:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1784949061.047275:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000fff5e130. 00000800:00000010:0.0:1784949061.047280:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000020:00000040:3.0:1784949061.047286:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000400:00000200:0.0:1784949061.047290:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949061.047297:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949061.047303:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949061.047311:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000400:00000010:0.0:1784949061.047314:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000100:00000001:0.0:1784949061.047320:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949061.047322:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949061.047329:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949061.047342:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949061.047347:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949061.047351:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949061.047357:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949061.047376:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a800 00000400:00000200:0.0:1784949061.047385:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8cfd [1] + 224 00000800:00000001:0.0:1784949061.047392:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949061.047400:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949061.047404:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949061.047410:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949061.047414:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949061.047418:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949061.047429:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949061.047488:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949061.047497:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949061.047503:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.047515:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949061.047522:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949061.047531:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949061.047534:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949061.047538:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002e5b8264. 00000400:00000200:1.0:1784949061.047545:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784949061.047548:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784949061.047552:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949061.047561:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949061.047574:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949061.047584:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949061.047586:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.047589:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949061.047593:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949061.047596:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949061.047599:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949061.047602:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949061.047612:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=78 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949061.047617:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=78 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949061.047620:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949061.047622:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949061.047626:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949061.047632:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949061.047641:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949061.047645:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949061.047647:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949061.047649:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949061.047651:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949061.047655:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949061.047663:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949061.047667:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949061.047669:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949061.047673:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949061.047681:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f59951e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289671680:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949061.047686:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.047688:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949061.047691:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f59951e9 x1871651289671680/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949061.047698:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949061.047701:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949061.047704:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:1.0:1784949061.047709:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949061.047711:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949061.047714:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949061.047717:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949061.047720:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f95b4924. 02000000:00000001:1.0:1784949061.047725:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949061.047727:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949061.047729:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f59951e9. 00000100:00000001:1.0:1784949061.047732:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949061.047735:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.047737:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949061.047742:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949061.047744:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949061.047747:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949061.047749:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949061.088050:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306000427, at 4306000428 00000100:00000040:2.0:1784949061.088054:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306000427, at 4306000428 00000100:00000040:3.0:1784949061.088060:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306000427, at 4306000428 00000100:00000010:1.0:1784949061.088065:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:0.0:1784949061.088086:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306000427, at 4306000428 00000100:00000010:2.0:1784949061.088087:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:3.0:1784949061.088090:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:0.0:1784949061.088109:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949061.876004:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949061.876012:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949061.876019:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949061.876022:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949061.876025:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949061.876028:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949061.876032:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949061.876034:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949061.876036:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949061.876038:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949061.876135:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949061.876142:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949061.876145:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949062.068029:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949062.068048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949062.068051:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949062.068067:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949062.068070:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949062.068074:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949062.068076:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949062.900035:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949062.900045:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949062.900053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949062.900055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949062.900187:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949062.900194:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949062.900198:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949063.092189:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949063.092205:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949063.092208:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949063.092214:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949063.092217:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949063.092221:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949063.092224:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949063.924027:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949063.924037:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949063.924045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949063.924048:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949063.924200:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949063.924211:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949063.924216:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949064.116091:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949064.116106:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949064.116109:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949064.116119:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949064.116124:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949064.116129:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949064.116133:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949064.948040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949064.948052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949064.948064:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949064.948066:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949064.948202:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949064.948209:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949064.948214:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949065.140151:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949065.140175:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949065.140179:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949065.140187:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949065.140192:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949065.140200:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949065.140205:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949065.972037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949065.972047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949065.972054:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949065.972056:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949065.972152:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949065.972159:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949065.972163:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949066.164041:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:2.0:1784949066.164052:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949066.164060:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949066.164062:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949066.164067:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784949066.164073:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000aaa80bc3. 00000100:00000001:0.0:1784949066.164075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949066.164080:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1784949066.164085:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949066.164087:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949066.164096:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949066.164097:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949066.164104:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949066.164110:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949066.164120:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002f91e44d. 00000100:00000001:2.0:1784949066.164129:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.164142:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949066.164157:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949066.164159:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.164166:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aaa80bc3] to pc [ptlrpcd_00_01+1] req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949066.164200:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949066.164204:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949066.164209:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949066.164214:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949066.164218:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949066.164221:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064fd9c10. 00000020:00000040:2.0:1784949066.164225:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949066.164228:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949066.164231:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949066.164234:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.164236:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1784949066.164240:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000de6364cf. 00000100:00000001:0.0:1784949066.164250:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949066.164252:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.164253:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949066.164264:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949066.164266:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.164268:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1784949066.164289:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949066.164291:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949066.164298:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.164321:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064fd9c10] to pc [ptlrpcd_00_02+2] req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949066.164354:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949066.164364:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949066.164371:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949066.164376:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949066.164379:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949066.164383:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000aaa80bc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289671808:0@lo:400: 00000100:00000001:2.0:1784949066.164386:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.164392:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1784949066.164400:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11343) 02000000:00000001:1.0:1784949066.164421:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949066.164426:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949066.164430:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949066.164434:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949066.164454:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1784949066.164456:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 00000100:00000001:0.0:1784949066.164457:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784949066.164464:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949066.164472:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949066.164473:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e5b8264. 00000100:00000001:3.0:1784949066.164480:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784949066.164489:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00000001:3.0:1784949066.164491:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949066.164497:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949066.164508:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949066.164512:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289671808, portal 25 02000000:00000001:0.0:1784949066.164513:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949066.164521:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1784949066.164523:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949066.164525:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562023040 : -115544147528576 : ffff96e9c8217480) 02000000:00000001:0.0:1784949066.164532:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949066.164543:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949066.164545:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064fd9c10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289671936:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949066.164574:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949066.164582:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949066.164584:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1784949066.164591:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784949066.164593:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949066.164602:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 02000000:00000001:0.0:1784949066.164604:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000200:1.0:1784949066.164614:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289671808, offset 0 02000000:00000010:0.0:1784949066.164618:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949066.164623:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949066.164627:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:0.0:1784949066.164637:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000047084e78. 00000400:00000010:0.0:1784949066.164648:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000400:00000200:1.0:1784949066.164648:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:0.0:1784949066.164667:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289671936, portal 10 00000100:00000001:0.0:1784949066.164672:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949066.164679:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562016768 : -115544147534848 : ffff96e9c8215c00) 00000400:00000200:1.0:1784949066.164680:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a880 00000400:00000200:1.0:1784949066.164705:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8305 [2] + 0 00000100:00000040:0.0:1784949066.164706:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949066.164729:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d40cdb47. 00000400:00000200:1.0:1784949066.164742:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000341aff9b 00000100:00000001:0.0:1784949066.164744:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949066.164750:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949066.164754:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:0.0:1784949066.164755:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000553902b5. 00000100:00000200:0.0:1784949066.164771:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289671936, offset 0 00000400:00000200:1.0:1784949066.164782:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949066.164796:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949066.164807:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000341aff9b 00000400:00000010:1.0:1784949066.164811:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000341aff9b. 00000100:00000001:1.0:1784949066.164820:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949066.164823:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1784949066.164827:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:1.0:1784949066.164830:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000222ac126 x1871651289671808 msgsize 224 00000100:00100000:1.0:1784949066.164843:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949066.164845:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784949066.164846:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949066.164858:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949066.164863:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949066.164868:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949066.164876:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949066.164883:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3573:3573:241:4294967295] 192.168.202.137@tcp LPNI seq info [2298:2298:8:4294967295] 00000400:00000200:0.0:1784949066.164951:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949066.164967:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949066.164970:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000010:0.0:1784949066.164979:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000595fa8f8. 00000100:00100000:2.0:1784949066.164980:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289671808 00000400:00000200:1.0:1784949066.164981:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784949066.164991:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 02000000:00000001:2.0:1784949066.164998:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949066.165001:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949066.165006:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:1.0:1784949066.165011:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784949066.165012:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949066.165016:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949066.165037:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.165041:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949066.165054:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949066.165055:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289671808 00000800:00000200:0.0:1784949066.165056:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:2.0:1784949066.165070:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:0.0:1784949066.165078:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000595fa8f8 type 1, nob 320 niov 1 nkiov 1 00000020:00000040:2.0:1784949066.165083:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949066.165097:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949066.165098:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949066.165104:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:2.0:1784949066.165109:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949066.165112:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.165124:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306005547 00000020:00000001:2.0:1784949066.165124:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949066.165138:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949066.165138:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000001:2.0:1784949066.165139:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949066.165151:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949066.165156:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:0.0:1784949066.165157:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2152 usecs (3 jiffies) 00000100:00000001:1.0:1784949066.165166:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306005547 00000020:00000010:2.0:1784949066.165170:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:0.0:1784949066.165171:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306005508 00000100:00000010:1.0:1784949066.165174:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:2.0:1784949066.165185:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949066.165191:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306005547 00000100:00000001:1.0:1784949066.165192:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:2.0:1784949066.165196:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000001:1.0:1784949066.165210:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306005547 00000100:00000001:0.0:1784949066.165213:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306005508 00000100:00000010:1.0:1784949066.165220:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000020:00000010:2.0:1784949066.165221:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e5260462. 00000100:00000001:0.0:1784949066.165223:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306005547 00000100:00000001:1.0:1784949066.165232:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949066.165240:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306005508 00000020:00000010:2.0:1784949066.165240:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004f0879c9. 00000100:00000001:1.0:1784949066.165242:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306005547 00000100:00000010:1.0:1784949066.165257:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784949066.165258:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306005547 00000020:00000010:2.0:1784949066.165258:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784949066.165268:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949066.165278:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306005508 00000100:00000040:2.0:1784949066.165284:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949066.165286:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949066.165287:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306005547 00000100:00000001:1.0:1784949066.165303:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949066.165305:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949066.165307:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949066.165310:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949066.165312:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949066.165321:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.165322:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949066.165323:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949066.165335:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949066.165339:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949066.165342:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.165348:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949066.165348:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949066.165350:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000595fa8f8. 00000100:00000001:0.0:1784949066.165353:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949066.165372:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949066.165374:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949066.165377:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949066.165381:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949066.165385:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949066.165392:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949066.165393:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949066.165394:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949066.165398:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949066.165435:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949066.165438:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553902b5 00000100:00000001:0.0:1784949066.165441:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:3.0:1784949066.165445:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000553902b5. 00000100:00000001:0.0:1784949066.165449:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949066.165456:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949066.165458:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949066.165465:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949066.165473:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:3.0:1784949066.165474:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949066.165479:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949066.165489:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949066.165498:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 768 00000800:00000001:0.0:1784949066.165502:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949066.165510:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949066.165516:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.165517:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000800:00000001:0.0:1784949066.165521:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949066.165524:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949066.165537:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949066.165539:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949066.165539:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165125200 : -115539544426416 : ffff96eada7f3850) 00000800:00000001:0.0:1784949066.165557:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949066.165561:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.165565:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000222ac126 x1871651289671808/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:727/0 lens 224/0 e 0 to 0 dl 1784949077 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949066.165568:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949066.165583:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949066.165608:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949066.165616:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643a900 00000020:00000010:2.0:1784949066.165622:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000400:00000200:0.0:1784949066.165632:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 167696 00000100:00000001:2.0:1784949066.165634:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949066.165645:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000800:00000001:0.0:1784949066.165652:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784949066.165660:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000222ac126 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289671808:12345-0@lo:400: 00000800:00000001:0.0:1784949066.165664:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949066.165674:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289671808 00000400:00000200:0.0:1784949066.165675:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949066.165684:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784949066.165689:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949066.165695:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949066.165699:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:2.0:1784949066.165703:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949066.165705:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:2.0:1784949066.165710:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000010:0.0:1784949066.165715:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000546ff4bc. 00000020:00000001:2.0:1784949066.165717:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000040:0.0:1784949066.165725:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000546ff4bc x1871651289671936 msgsize 224 00000020:00000001:2.0:1784949066.165729:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949066.165737:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949066.165742:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:2.0:1784949066.165746:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949066.165751:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949066.165754:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949066.165757:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949066.165761:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949066.165764:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949066.165765:0:337112:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:2.0:1784949066.165771:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a6fbdd8b. 02000000:00000001:2.0:1784949066.165777:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949066.165779:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949066.165780:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949066.165787:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949066.165789:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:2.0:1784949066.165792:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289671808 00010000:00000001:2.0:1784949066.165802:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949066.165810:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000222ac126 x1871651289671808/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:727/0 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949066.165822:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949066.165826:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949066.165826:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949066.165834:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289671936 00000100:00001000:2.0:1784949066.165836:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=88 v=5 (1 1 1 1) 02000000:00000001:3.0:1784949066.165843:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949066.165847:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949066.165848:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949066.165855:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.165857:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949066.165867:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949066.165867:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949066.165872:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00100000:3.0:1784949066.165875:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289671936 02000000:00000001:2.0:1784949066.165877:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949066.165883:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949066.165885:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1784949066.165888:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000040:2.0:1784949066.165896:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:3.0:1784949066.165897:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:2.0:1784949066.165939:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000200:2.0:1784949066.165946:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289671808, offset 224 00000400:00000200:2.0:1784949066.165954:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000040:3.0:1784949066.165960:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:2.0:1784949066.165970:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:3.0:1784949066.165974:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949066.165984:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784949066.165993:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a880 00000100:00000001:3.0:1784949066.165996:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949066.166009:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000400:00000200:2.0:1784949066.166015:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8d19 [1] + 224 00000100:00000001:3.0:1784949066.166019:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949066.166030:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000400:00000200:2.0:1784949066.166035:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949066.166043:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000053ab16eb. 00000400:00000200:2.0:1784949066.166049:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:3.0:1784949066.166055:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e7178061. 00000100:00000001:2.0:1784949066.166061:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949066.166063:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000200:2.0:1784949066.166076:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949066.166078:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949066.166085:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949066.166089:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949066.166095:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949066.166106:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949066.166112:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000040:2.0:1784949066.166117:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949066.166118:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949066.166141:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949066.166157:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949066.166162:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949066.166162:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949066.166172:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 744 00000100:00000040:3.0:1784949066.166177:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000200:2.0:1784949066.166179:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000100:00000001:3.0:1784949066.166181:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554975232 : -115544154576384 : ffff96e9c7b5ea00) 00000400:00000010:2.0:1784949066.166189:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:2.0:1784949066.166196:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:3.0:1784949066.166198:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000546ff4bc x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:727/0 lens 224/0 e 0 to 0 dl 1784949077 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949066.166201:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949066.166211:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949066.166224:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784949066.166232:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949066.166235:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:2.0:1784949066.166236:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.166241:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784949066.166246:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949066.166247:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1784949066.166248:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000040:2.0:1784949066.166259:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000222ac126 x1871651289671808/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:727/0 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949066.166263:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949066.166263:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1784949066.166276:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949066.166287:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002e5b8264. 00000020:00000010:3.0:1784949066.166289:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000400:00000200:1.0:1784949066.166299:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000100:00100000:2.0:1784949066.166301:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000222ac126 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289671808:12345-0@lo:400: Request processed in 688us (1463us total) trans 0 rc 0/0 00000400:00000010:1.0:1784949066.166305:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784949066.166321:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1784949066.166322:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000546ff4bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289671936:12345-192.168.202.137@tcp:400: 00000100:00100000:2.0:1784949066.166331:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 768 00000100:00000200:1.0:1784949066.166333:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949066.166344:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:3.0:1784949066.166350:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289671936 00000100:00000001:2.0:1784949066.166356:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:3.0:1784949066.166362:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949066.166367:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:3.0:1784949066.166373:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000010:2.0:1784949066.166382:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004f0879c9. 00000020:00000001:3.0:1784949066.166384:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949066.166402:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000200:1.0:1784949066.166426:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949066.166432:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949066.166441:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000010:2.0:1784949066.166444:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e5260462. 00000020:00000001:3.0:1784949066.166458:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949066.166465:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1784949066.166466:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784949066.166467:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949066.166469:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949066.166484:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949066.166485:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:3.0:1784949066.166485:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000010:2.0:1784949066.166494:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a6fbdd8b. 02000000:00000001:1.0:1784949066.166495:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:3.0:1784949066.166495:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949066.166511:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:2.0:1784949066.166513:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:3.0:1784949066.166513:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.166517:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949066.166521:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949066.166527:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949066.166528:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000200:2.0:1784949066.166532:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949066.166542:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949066.166548:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00001000:1.0:1784949066.166549:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=88 v=5 (5 5 5 5) 00000400:00000010:2.0:1784949066.166550:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000603fb577. 02000000:00000010:3.0:1784949066.166564:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b809337d. 00000100:00001000:1.0:1784949066.166566:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=88 v=5 (1 1 1 1) 00000400:00000010:2.0:1784949066.166568:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000031b88578. 02000000:00000001:3.0:1784949066.166579:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.166582:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949066.166589:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949066.166589:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949066.166615:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949066.166615:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949066.166632:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949066.166634:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289671936 00000100:00000001:1.0:1784949066.166654:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949066.166654:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:1.0:1784949066.166672:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:3.0:1784949066.166673:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000546ff4bc x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:727/0 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949066.166747:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784949066.166754:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949066.166755:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949066.166767:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949066.166767:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949066.166780:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=83 v=5 (1 1 1 1) 00000100:00000040:1.0:1784949066.166785:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949066.166793:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949066.166808:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00100000:1.0:1784949066.166818:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000aaa80bc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289671808:0@lo:400: 00000100:00000001:3.0:1784949066.166819:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949066.166833:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784949066.166837:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949066.166838:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.166858:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949066.166860:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949066.166874:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000aaa80bc3 x1871651289671808/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949066.166876:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949066.166889:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000100:00000001:1.0:1784949066.166893:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000200:3.0:1784949066.166897:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289671936, offset 224 02000000:00000001:1.0:1784949066.166898:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949066.166925:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:1.0:1784949066.166931:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949066.166933:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949066.166935:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949066.166937:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949066.166941:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002f91e44d. 00000400:00000200:3.0:1784949066.166948:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949066.166949:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949066.166954:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949066.166956:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000aaa80bc3. 00000100:00000001:1.0:1784949066.166963:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949066.166969:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949066.166973:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949066.166977:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949066.166990:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1784949066.166990:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3574:3574:241:4294967295] 192.168.202.137@tcp LPNI seq info [2299:2299:8:4294967295] 00000100:00000001:1.0:1784949066.166997:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949066.167007:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949066.167011:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949066.167027:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949066.167037:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949066.167043:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949066.167050:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949066.167057:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949066.167060:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949066.167077:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949066.167081:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949066.167084:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949066.167086:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949066.167088:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949066.167094:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000546ff4bc x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:727/0 lens 224/224 e 0 to 0 dl 1784949077 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949066.167114:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000546ff4bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289671936:12345-192.168.202.137@tcp:400: Request processed in 878us (1386us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949066.167130:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 744 00000100:00000040:3.0:1784949066.167136:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949066.167141:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949066.167144:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949066.167149:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e7178061. 00000020:00000010:3.0:1784949066.167157:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949066.167164:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000053ab16eb. 00000020:00000040:3.0:1784949066.167171:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949066.167177:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949066.167242:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949066.167248:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000800:00000001:3.0:1784949066.167254:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949066.167255:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949066.167269:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949066.167275:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949066.167276:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000400:00000010:0.0:1784949066.167283:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000800:00000001:3.0:1784949066.167285:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949066.167293:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784949066.167294:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949066.167300:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949066.167309:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949066.167325:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643a900 00000400:00000200:3.0:1784949066.167333:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8d21 [1] + 224 00000800:00000001:3.0:1784949066.167341:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949066.167351:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949066.167354:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949066.167359:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949066.167365:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949066.167369:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949066.167381:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949066.167426:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949066.167435:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949066.167439:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949066.167483:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949066.167490:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949066.167498:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949066.167502:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949066.167507:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000047084e78. 00000400:00000200:0.0:1784949066.167515:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:0.0:1784949066.167518:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000100:00000001:0.0:1784949066.167523:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949066.167531:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949066.167551:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949066.167565:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949066.167568:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949066.167573:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949066.167577:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949066.167581:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949066.167584:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949066.167588:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949066.167597:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=83 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949066.167603:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=83 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949066.167609:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949066.167612:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949066.167616:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949066.167624:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949066.167637:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949066.167643:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949066.167646:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949066.167650:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949066.167654:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949066.167660:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949066.167674:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949066.167679:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949066.167682:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949066.167689:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949066.167717:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064fd9c10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289671936:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949066.167726:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949066.167731:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949066.167735:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064fd9c10 x1871651289671936/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949066.167749:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949066.167754:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949066.167758:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949066.167766:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949066.167770:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949066.167773:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949066.167777:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949066.167780:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000de6364cf. 02000000:00000001:0.0:1784949066.167788:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949066.167791:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949066.167794:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064fd9c10. 00000100:00000001:0.0:1784949066.167800:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949066.167801:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949066.167805:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949066.167813:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949066.167817:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949066.167821:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949066.167823:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949066.208004:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306005547, at 4306005548 00000100:00000010:3.0:1784949066.208025:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:2.0:1784949066.208063:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306005547, at 4306005548 00000100:00000040:1.0:1784949066.208074:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306005547, at 4306005548 00000100:00000040:0.0:1784949066.208079:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306005547, at 4306005548 00000100:00000010:2.0:1784949066.208082:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:1.0:1784949066.208091:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:0.0:1784949066.208098:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000400:00000010:1.0:1784949066.356049:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000048122e2d. 00000400:00000010:1.0:1784949066.356079:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c4fd57a4. 00000400:00000010:1.0:1784949066.356082:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004656e9af. 00000400:00000010:1.0:1784949066.356084:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004bacba68. 00000400:00000010:1.0:1784949066.356085:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f2e2ddd3. 00000400:00000010:1.0:1784949066.356088:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000071cfa864. 00000400:00000010:1.0:1784949066.356090:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f3516823. 00000400:00000010:1.0:1784949066.356092:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d9c0bef1. 00000400:00000010:1.0:1784949066.356094:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007a213df0. 00000400:00000010:1.0:1784949066.356096:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000056f20560. 00000400:00000010:1.0:1784949066.356098:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000619517d8. 00000400:00000010:1.0:1784949066.356100:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004c587164. 00000400:00000010:1.0:1784949066.356102:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004cc664c0. 00000400:00000010:1.0:1784949066.356106:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000049855afc. 00000400:00000010:1.0:1784949066.356108:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a7d88830. 00000400:00000010:1.0:1784949066.356109:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000060798c14. 00000400:00000010:1.0:1784949066.356111:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000062d84ae9. 00000400:00000010:1.0:1784949066.356112:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000473e84ce. 00000400:00000010:1.0:1784949066.356114:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000775d3d92. 00000400:00000010:1.0:1784949066.356115:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e4a8defd. 00000400:00000010:1.0:1784949066.356117:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000074e9a075. 00000400:00000010:1.0:1784949066.356119:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a816c742. 00000400:00000010:1.0:1784949066.356120:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006aad92e0. 00000400:00000010:1.0:1784949066.356122:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004f0f2833. 00000400:00000010:1.0:1784949066.356123:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000092f0a178. 00000400:00000010:1.0:1784949066.356126:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000890c5791. 00000400:00000010:1.0:1784949066.356128:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000064d54234. 00000400:00000010:1.0:1784949066.356129:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000005df51d2. 00000400:00000010:1.0:1784949066.356131:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000009797c23. 00000400:00000010:1.0:1784949066.356133:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b9d99f33. 00000400:00000010:1.0:1784949066.356135:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000092c40cf9. 00000400:00000010:1.0:1784949066.356136:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d4c102f3. 00000400:00000010:1.0:1784949066.356142:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000934e2586. 00000400:00000010:1.0:1784949066.356144:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b03d8085. 00000800:00000001:0.0:1784949066.575493:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949066.575519:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949066.575527:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949066.575530:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949066.996000:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949066.996009:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949066.996017:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949066.996018:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949066.996071:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949066.996077:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949066.996081:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949067.188105:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949067.188127:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949067.188130:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949067.188138:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949067.188143:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949067.188148:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949067.188151:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949068.019973:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949068.019979:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949068.019983:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949068.019984:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949068.020076:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949068.020085:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949068.020089:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949068.212017:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949068.212034:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949068.212036:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949068.212041:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949068.212044:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949068.212049:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949068.212051:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949069.044049:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949069.044061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949069.044072:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949069.044075:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949069.044204:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949069.044213:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949069.044217:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949069.236053:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949069.236068:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949069.236070:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949069.236079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949069.236083:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949069.236088:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949069.236092:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949070.068026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949070.068036:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949070.068045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949070.068047:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949070.068163:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949070.068173:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949070.068178:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949070.260154:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949070.260177:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949070.260180:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949070.260189:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949070.260193:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949070.260200:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949070.260205:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949071.092054:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949071.092067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949071.092078:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949071.092083:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949071.092088:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949071.092093:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949071.092098:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949071.092101:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949071.092105:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949071.092109:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949071.092245:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949071.092252:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949071.092257:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949071.284077:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949071.284094:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949071.284095:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949071.284104:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949071.284109:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949071.284110:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949071.284114:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.284124:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.284129:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949071.284139:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bb1b4c23. 00000020:00000040:0.0:1784949071.284149:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949071.284155:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949071.284159:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949071.284165:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949071.284174:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f64a2f6c. 00000100:00000001:0.0:1784949071.284183:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949071.284194:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949071.284209:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949071.284212:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949071.284217:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb1b4c23] to pc [ptlrpcd_00_03+3] req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949071.284262:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949071.284265:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949071.284270:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949071.284273:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949071.284278:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949071.284283:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000047c147a0. 00000020:00000040:0.0:1784949071.284286:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949071.284289:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949071.284291:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949071.284293:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949071.284295:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949071.284302:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000000d1fbdb0. 00000100:00100000:1.0:1784949071.284302:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:1.0:1784949071.284319:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949071.284321:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.284327:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949071.284332:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949071.284333:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949071.284334:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949071.284338:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949071.284345:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949071.284349:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.284352:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949071.284389:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949071.284409:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949071.284415:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.284418:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949071.284425:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949071.284431:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047c147a0] to pc [ptlrpcd_00_00+0] req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949071.284433:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949071.284440:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949071.284456:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949071.284473:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000bb1b4c23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289672064:0@lo:400: 00000100:00000001:1.0:1784949071.284481:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949071.284489:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949071.284491:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949071.284494:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949071.284499:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949071.284504:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 00000100:00000040:0.0:1784949071.284508:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11348) 02000000:00000001:1.0:1784949071.284508:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949071.284520:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e5b8264. 00000400:00000010:1.0:1784949071.284531:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00000200:1.0:1784949071.284541:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289672064, portal 25 00000100:00000001:1.0:1784949071.284547:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949071.284547:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949071.284556:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554979712 : -115544154571904 : ffff96e9c7b5fb80) 00000100:00000001:2.0:1784949071.284559:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949071.284564:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949071.284565:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949071.284578:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949071.284580:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949071.284581:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949071.284590:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949071.284602:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949071.284612:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000200:1.0:1784949071.284623:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289672064, offset 0 00000100:00000001:2.0:1784949071.284635:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949071.284639:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949071.284649:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784949071.284653:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949071.284658:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949071.284663:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949071.284673:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000047c147a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289672192:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784949071.284677:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949071.284693:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949071.284701:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1784949071.284705:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643a980 02000000:00000001:2.0:1784949071.284708:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949071.284716:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949071.284722:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x832d [2] + 0 02000000:00000010:2.0:1784949071.284727:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 00000400:00000010:1.0:1784949071.284728:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000010aeeaab. 02000000:00000001:2.0:1784949071.284745:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949071.284747:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f29bca31 00000400:00000010:2.0:1784949071.284754:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002f32d921. 00000400:00000200:1.0:1784949071.284760:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1784949071.284766:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000400:00000200:1.0:1784949071.284771:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:2.0:1784949071.284779:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289672192, portal 10 00000400:00000200:1.0:1784949071.284780:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f29bca31 00000400:00000010:1.0:1784949071.284794:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f29bca31. 00000100:00000001:2.0:1784949071.284794:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949071.284808:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554976128 : -115544154575488 : ffff96e9c7b5ed80) 00000100:00000001:1.0:1784949071.284812:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949071.284816:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784949071.284821:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000bc66e638 x1871651289672064 msgsize 224 00000100:00000040:2.0:1784949071.284823:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949071.284849:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949071.284862:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949071.284867:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949071.284873:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949071.284876:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000100:00000200:2.0:1784949071.284879:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289672192, offset 0 00000100:00000001:1.0:1784949071.284883:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949071.284888:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949071.284894:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784949071.284897:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:1.0:1784949071.284902:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949071.284931:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949071.284942:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949071.284944:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949071.284949:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949071.284966:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949071.284973:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949071.284988:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784949071.284991:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.284998:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784949071.284998:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949071.285009:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3575:3575:241:4294967295] 192.168.202.137@tcp LPNI seq info [2300:2300:8:4294967295] 00000100:00000001:0.0:1784949071.285019:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949071.285028:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289672064 00000400:00000200:2.0:1784949071.285030:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:0.0:1784949071.285035:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949071.285041:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949071.285046:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949071.285047:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949071.285048:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949071.285059:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e1f7df0e. 02000000:00000001:0.0:1784949071.285061:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.285062:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306010667 00000800:00000200:2.0:1784949071.285076:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:0.0:1784949071.285079:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289672064 00000100:00000010:1.0:1784949071.285083:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000001:0.0:1784949071.285086:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:2.0:1784949071.285098:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000040:0.0:1784949071.285099:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949071.285108:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949071.285108:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000800:00000200:2.0:1784949071.285109:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e1f7df0e type 1, nob 320 niov 1 nkiov 1 00000020:00000040:0.0:1784949071.285118:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784949071.285126:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949071.285129:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306010667 00000020:00000001:0.0:1784949071.285130:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784949071.285141:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:2.0:1784949071.285141:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.285143:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949071.285158:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076985c84. 00000100:00000040:2.0:1784949071.285167:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2155 usecs (3 jiffies) 00000100:00000001:0.0:1784949071.285168:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949071.285168:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949071.285189:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306010628 00000020:00000010:0.0:1784949071.285190:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076985c84. 00000100:00000001:1.0:1784949071.285192:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306010667 00000100:00000001:2.0:1784949071.285196:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306010667 00000100:00000001:2.0:1784949071.285209:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306010628 00000100:00000010:1.0:1784949071.285210:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000020:00000010:0.0:1784949071.285213:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dc7a25c6. 00000100:00000001:2.0:1784949071.285226:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306010667 00000100:00000001:2.0:1784949071.285234:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306010628 00000020:00000010:0.0:1784949071.285237:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007194bf82. 00000100:00000001:1.0:1784949071.285237:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949071.285251:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306010667 00000020:00000010:0.0:1784949071.285252:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949071.285253:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306010667 00000100:00000010:1.0:1784949071.285262:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:2.0:1784949071.285271:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306010628 00000100:00000001:1.0:1784949071.285272:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000040:0.0:1784949071.285275:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949071.285283:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949071.285285:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306010667 00000100:00000001:0.0:1784949071.285287:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949071.285294:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.285295:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949071.285303:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949071.285307:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.285307:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949071.285321:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.285329:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949071.285334:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.285338:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949071.285343:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949071.285359:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949071.285389:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:3.0:1784949071.285390:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949071.285395:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949071.285399:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949071.285400:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:3.0:1784949071.285404:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e1f7df0e. 00000100:00000001:2.0:1784949071.285413:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949071.285421:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949071.285423:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949071.285425:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.285430:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:1.0:1784949071.285437:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949071.285439:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784949071.285441:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949071.285442:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949071.285449:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949071.285455:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000100:00100000:0.0:1784949071.285461:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 769 00000400:00000010:3.0:1784949071.285465:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000100:00000040:0.0:1784949071.285470:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949071.285478:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165112912 : -115539544438704 : ffff96eada7f0850) 00000400:00000200:1.0:1784949071.285479:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949071.285480:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949071.285494:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949071.285502:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bc66e638 x1871651289672064/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:732/0 lens 224/0 e 0 to 0 dl 1784949082 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949071.285516:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643aa00 00000400:00000200:1.0:1784949071.285555:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 167920 00000800:00000001:1.0:1784949071.285585:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949071.285592:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:1.0:1784949071.285602:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949071.285603:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1784949071.285604:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076985c84. 00000100:00000040:3.0:1784949071.285621:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949071.285623:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949071.285624:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949071.285641:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076985c84. 00000400:00000200:1.0:1784949071.285656:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:0.0:1784949071.285671:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bc66e638 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289672064:12345-0@lo:400: 00000100:00000001:3.0:1784949071.285677:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949071.285680:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289672064 00000100:00000001:1.0:1784949071.285681:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949071.285684:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:1.0:1784949071.285689:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:0.0:1784949071.285694:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000010:1.0:1784949071.285697:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb5afc66. 00000020:00000001:0.0:1784949071.285701:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:1.0:1784949071.285706:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000eb5afc66 x1871651289672192 msgsize 224 00000020:00000001:0.0:1784949071.285707:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949071.285716:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949071.285723:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00100000:1.0:1784949071.285735:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:0.0:1784949071.285737:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949071.285741:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949071.285746:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949071.285749:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949071.285752:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.285753:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:0.0:1784949071.285755:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949071.285760:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000001:1.0:1784949071.285763:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:0.0:1784949071.285771:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:1.0:1784949071.285774:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784949071.285779:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009925ca9f. 02000000:00000001:0.0:1784949071.285784:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.285788:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949071.285793:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949071.285797:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289672064 00000100:00000001:3.0:1784949071.285802:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:0.0:1784949071.285803:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00100000:3.0:1784949071.285809:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289672192 00010000:00000200:0.0:1784949071.285814:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bc66e638 x1871651289672064/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:732/0 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949071.285816:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949071.285822:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949071.285828:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949071.285838:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:3.0:1784949071.285842:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949071.285844:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949071.285854:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289672192 00000100:00001000:0.0:1784949071.285855:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=93 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949071.285865:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949071.285868:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:3.0:1784949071.285874:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000040:0.0:1784949071.285876:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949071.285885:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:3.0:1784949071.285885:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:0.0:1784949071.285894:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000040:3.0:1784949071.285895:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:0.0:1784949071.285897:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.285941:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949071.285947:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949071.285951:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00000200:0.0:1784949071.285957:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289672064, offset 224 00000020:00000001:3.0:1784949071.285966:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784949071.285970:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:3.0:1784949071.285980:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949071.285991:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949071.285993:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:3.0:1784949071.286004:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:3.0:1784949071.286012:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949071.286013:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643a980 00000020:00000010:3.0:1784949071.286024:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000400:00000200:0.0:1784949071.286033:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8d35 [1] + 224 00000020:00000010:3.0:1784949071.286043:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000fff5e130. 00000400:00000200:0.0:1784949071.286046:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949071.286055:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:3.0:1784949071.286056:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000da98738. 00000100:00000001:0.0:1784949071.286061:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949071.286068:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000200:0.0:1784949071.286070:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949071.286090:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:0.0:1784949071.286091:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949071.286102:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949071.286106:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949071.286114:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949071.286122:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949071.286125:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949071.286127:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949071.286133:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949071.286146:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:0.0:1784949071.286151:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000100:00000001:3.0:1784949071.286152:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949071.286158:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949071.286163:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949071.286164:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784949071.286178:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949071.286179:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949071.286181:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949071.286188:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000001:1.0:1784949071.286189:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:0.0:1784949071.286195:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:1.0:1784949071.286197:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949071.286209:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002e5b8264. 00010000:00000001:0.0:1784949071.286210:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00100000:3.0:1784949071.286211:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 745 00000020:00000001:0.0:1784949071.286225:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949071.286228:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000100:00000040:3.0:1784949071.286232:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000020:00000001:0.0:1784949071.286237:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:1.0:1784949071.286237:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:3.0:1784949071.286243:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529549154432 : -115544160397184 : ffff96e9c75d1880) 00000100:00000001:1.0:1784949071.286253:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784949071.286254:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bc66e638 x1871651289672064/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:732/0 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949071.286268:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949071.286272:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000eb5afc66 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:732/0 lens 224/0 e 0 to 0 dl 1784949082 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949071.286307:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bc66e638 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289672064:12345-0@lo:400: Request processed in 694us (1464us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949071.286385:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 769 00000100:00000040:0.0:1784949071.286410:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949071.286418:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:1.0:1784949071.286420:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949071.286420:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949071.286423:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949071.286432:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000020:00000010:0.0:1784949071.286435:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007194bf82. 00000100:00000001:1.0:1784949071.286439:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949071.286441:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949071.286447:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949071.286451:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:3.0:1784949071.286456:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000100:00000001:1.0:1784949071.286458:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:0.0:1784949071.286460:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dc7a25c6. 02000000:00000001:1.0:1784949071.286468:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949071.286473:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949071.286477:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949071.286479:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00100000:3.0:1784949071.286488:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000eb5afc66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289672192:12345-192.168.202.137@tcp:400: 02000000:00000001:1.0:1784949071.286489:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949071.286492:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:0.0:1784949071.286503:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009925ca9f. 00000100:00001000:1.0:1784949071.286507:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=93 v=5 (5 5 5 5) 00000100:00000200:3.0:1784949071.286508:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289672192 02000000:00000001:0.0:1784949071.286513:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:3.0:1784949071.286521:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949071.286523:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784949071.286525:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=93 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949071.286527:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949071.286535:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949071.286540:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949071.286542:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784949071.286545:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949071.286549:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949071.286553:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00010000:00000001:1.0:1784949071.286555:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:0.0:1784949071.286556:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000047084e78. 00010000:00000001:1.0:1784949071.286571:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949071.286571:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:0.0:1784949071.286575:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d4d2d632. 00000020:00000001:3.0:1784949071.286586:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949071.286587:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949071.286607:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784949071.286609:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949071.286617:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949071.286620:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949071.286626:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1784949071.286628:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.286629:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949071.286644:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:1.0:1784949071.286645:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949071.286658:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000bb1b4c23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289672064:0@lo:400: 00000100:00000001:3.0:1784949071.286659:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949071.286664:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949071.286666:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949071.286667:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949071.286678:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000bb1b4c23 x1871651289672064/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784949071.286682:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001d4383fd. 00000100:00000001:1.0:1784949071.286683:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949071.286686:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949071.286691:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:3.0:1784949071.286692:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949071.286697:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949071.286701:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784949071.286701:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949071.286706:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949071.286716:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:3.0:1784949071.286717:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949071.286719:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f64a2f6c. 02000000:00000001:1.0:1784949071.286726:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00010000:00000040:3.0:1784949071.286729:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289672192 02000000:00000001:1.0:1784949071.286730:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949071.286733:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000bb1b4c23. 00010000:00000001:3.0:1784949071.286741:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949071.286750:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000eb5afc66 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:732/0 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949071.286751:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949071.286755:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949071.286770:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949071.286784:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949071.286790:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949071.286798:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1784949071.286799:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949071.286803:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949071.286805:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949071.286812:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=88 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949071.286819:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949071.286822:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949071.286826:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949071.286829:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949071.286831:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949071.286834:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949071.286838:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949071.286843:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 00000100:00000200:3.0:1784949071.286847:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289672192, offset 224 00000400:00000200:3.0:1784949071.286854:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949071.286864:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949071.286871:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3576:3576:241:4294967295] 192.168.202.137@tcp LPNI seq info [2301:2301:8:4294967295] 00000400:00000200:3.0:1784949071.286882:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949071.286888:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949071.286893:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784949071.286898:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949071.286905:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949071.286943:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949071.286958:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949071.286962:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949071.286965:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949071.286967:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949071.286970:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949071.286974:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb5afc66 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:732/0 lens 224/224 e 0 to 0 dl 1784949082 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949071.286986:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000eb5afc66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289672192:12345-192.168.202.137@tcp:400: Request processed in 567us (1278us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949071.286997:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 745 00000100:00000040:3.0:1784949071.287001:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949071.287003:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949071.287005:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949071.287010:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000da98738. 00000020:00000010:3.0:1784949071.287014:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949071.287018:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000fff5e130. 00000020:00000040:3.0:1784949071.287022:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949071.287025:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949071.287194:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949071.287202:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:0.0:1784949071.287210:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949071.287219:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949071.287225:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000800:00000001:3.0:1784949071.287229:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:0.0:1784949071.287230:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000100:00000001:0.0:1784949071.287240:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949071.287244:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949071.287254:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949071.287263:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949071.287272:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949071.287284:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949071.287297:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643aa00 00000400:00000200:3.0:1784949071.287305:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8d3d [1] + 224 00000800:00000001:3.0:1784949071.287311:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949071.287321:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949071.287324:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949071.287328:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949071.287334:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949071.287339:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949071.287351:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949071.287401:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949071.287410:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949071.287415:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949071.287512:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949071.287518:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949071.287527:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949071.287530:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949071.287533:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002f32d921. 00000400:00000200:2.0:1784949071.287541:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000010:2.0:1784949071.287543:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:2.0:1784949071.287548:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949071.287555:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949071.287569:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949071.287579:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949071.287581:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949071.287584:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949071.287588:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949071.287590:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949071.287592:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949071.287595:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949071.287602:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=88 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949071.287606:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=88 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949071.287610:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949071.287612:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949071.287615:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949071.287620:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949071.287629:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949071.287633:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949071.287635:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949071.287637:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949071.287639:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949071.287643:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949071.287652:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949071.287655:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949071.287658:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949071.287661:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949071.287669:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000047c147a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289672192:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949071.287675:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949071.287677:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949071.287680:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000047c147a0 x1871651289672192/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949071.287688:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949071.287690:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949071.287693:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 02000000:00000001:2.0:1784949071.287698:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949071.287700:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949071.287702:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949071.287705:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949071.287708:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000000d1fbdb0. 02000000:00000001:2.0:1784949071.287711:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949071.287713:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949071.287715:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000047c147a0. 00000100:00000001:2.0:1784949071.287734:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949071.287735:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949071.287738:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949071.287743:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949071.287745:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949071.287748:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949071.287750:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949071.328054:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306010667, at 4306010668 00000100:00000040:1.0:1784949071.328069:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306010667, at 4306010668 00000100:00000010:3.0:1784949071.328071:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:0.0:1784949071.328094:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306010667, at 4306010668 00000100:00000010:1.0:1784949071.328094:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:2.0:1784949071.328100:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306010667, at 4306010668 00000100:00000010:0.0:1784949071.328121:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:2.0:1784949071.328127:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00010000:00000001:2.0:1784949072.115977:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949072.115983:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949072.115988:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949072.115989:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949072.116144:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949072.116154:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949072.116160:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949072.308019:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949072.308039:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949072.308041:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949072.308047:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949072.308051:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949072.308056:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949072.308059:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949073.140089:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949073.140098:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949073.140106:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949073.140107:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949073.140235:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949073.140241:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949073.140246:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949073.332124:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949073.332138:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949073.332140:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949073.332145:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949073.332150:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949073.332154:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949073.332157:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949074.164065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949074.164075:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949074.164083:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949074.164085:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949074.164157:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949074.164167:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949074.164172:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949074.355995:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949074.356008:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949074.356011:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949074.356016:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949074.356030:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949074.356035:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949074.356037:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949075.188042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949075.188051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949075.188087:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949075.188089:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949075.188183:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949075.188189:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949075.188195:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949075.380146:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949075.380164:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949075.380166:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949075.380172:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949075.380176:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949075.380180:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949075.380183:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949076.212046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949076.212056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949076.212063:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949076.212065:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949076.212162:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949076.212171:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949076.212175:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949076.404025:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949076.404042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949076.404044:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949076.404048:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949076.404052:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949076.404056:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.404058:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.404121:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949076.404131:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949076.404141:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000020:00000040:3.0:1784949076.404149:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949076.404155:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949076.404159:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949076.404166:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949076.404176:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:3.0:1784949076.404185:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.404195:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949076.404209:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949076.404212:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.404217:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006d868603] to pc [ptlrpcd_00_01+1] req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949076.404258:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949076.404261:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949076.404266:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949076.404271:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949076.404275:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949076.404279:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006e540bde. 00000020:00000040:3.0:1784949076.404282:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949076.404283:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949076.404285:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784949076.404289:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949076.404289:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949076.404300:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1784949076.404301:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949076.404315:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000040:1.0:1784949076.404342:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949076.404345:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.404363:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949076.404369:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949076.404376:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949076.404389:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949076.404392:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949076.404395:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949076.404397:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.404400:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949076.404402:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.404410:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e540bde] to pc [ptlrpcd_00_02+2] req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949076.404416:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289672320:0@lo:400: 00000100:00000001:2.0:1784949076.404417:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.404447:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949076.404462:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949076.404466:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949076.404468:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949076.404478:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 02000000:00000001:1.0:1784949076.404481:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949076.404482:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.404491:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11353) 00000400:00000010:1.0:1784949076.404492:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e5b8264. 00000400:00000010:1.0:1784949076.404516:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00000200:1.0:1784949076.404528:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289672320, portal 25 00000100:00000001:0.0:1784949076.404530:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949076.404533:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949076.404533:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949076.404542:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949076.404544:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554951680 : -115544154599936 : ffff96e9c7b58e00) 00000100:00000001:2.0:1784949076.404546:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.404550:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949076.404574:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949076.404577:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949076.404623:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949076.404635:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949076.404641:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000001:0.0:1784949076.404644:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:1.0:1784949076.404645:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289672320, offset 0 00000400:00000200:1.0:1784949076.404667:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949076.404669:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949076.404681:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949076.404687:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949076.404687:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:0.0:1784949076.404702:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949076.404708:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643aa80 00000100:00100000:0.0:1784949076.404716:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289672448:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784949076.404727:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x83c5 [2] + 0 00000100:00000001:0.0:1784949076.404743:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:1.0:1784949076.404744:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e8e1fecb. 02000000:00000001:0.0:1784949076.404758:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1784949076.404760:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000090a0d94e 02000000:00000001:0.0:1784949076.404766:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949076.404772:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949076.404775:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:0.0:1784949076.404780:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 00000400:00000200:1.0:1784949076.404789:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784949076.404791:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949076.404797:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000090a0d94e 00000400:00000010:0.0:1784949076.404800:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 00000400:00000010:1.0:1784949076.404802:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000090a0d94e. 00000100:00000001:1.0:1784949076.404815:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949076.404817:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:0.0:1784949076.404818:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00000040:1.0:1784949076.404822:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000ace3646f x1871651289672320 msgsize 224 00000100:00100000:1.0:1784949076.404827:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949076.404829:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949076.404853:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:0.0:1784949076.404857:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289672448, portal 10 00000100:00000001:0.0:1784949076.404862:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949076.404864:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554957056 : -115544154594560 : ffff96e9c7b5a300) 00000400:00000200:1.0:1784949076.404865:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784949076.404870:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:1.0:1784949076.404878:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784949076.404883:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949076.404884:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949076.404913:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949076.404918:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949076.404922:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002e7a7cb2. 00000100:00000200:0.0:1784949076.404926:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289672448, offset 0 00000400:00000200:0.0:1784949076.404943:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949076.404951:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949076.404958:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949076.404966:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949076.404967:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.404980:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784949076.404981:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289672320 02000000:00000001:2.0:1784949076.404994:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949076.404999:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.405000:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949076.405006:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.405011:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306015787 00000400:00000200:0.0:1784949076.405016:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000001:2.0:1784949076.405016:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949076.405029:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00100000:2.0:1784949076.405029:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289672320 00000400:00000200:0.0:1784949076.405033:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1784949076.405041:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949076.405046:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:0.0:1784949076.405048:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949076.405049:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1784949076.405053:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949076.405066:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:0.0:1784949076.405070:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784949076.405071:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306015787 00000020:00000001:2.0:1784949076.405075:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784949076.405087:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784949076.405089:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000400:00000200:0.0:1784949076.405090:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949076.405108:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949076.405109:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949076.405112:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3577:3577:241:4294967295] 192.168.202.137@tcp LPNI seq info [2302:2302:8:4294967295] 00000100:00000001:1.0:1784949076.405124:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306015787 00000020:00000010:2.0:1784949076.405127:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000036da8adc. 00000100:00000010:1.0:1784949076.405130:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:2.0:1784949076.405137:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949076.405143:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:2.0:1784949076.405145:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000036da8adc. 00000100:00000001:1.0:1784949076.405154:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306015787 00000400:00000200:0.0:1784949076.405155:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:2.0:1784949076.405158:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e5260462. 00000100:00000010:1.0:1784949076.405172:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000020:00000010:2.0:1784949076.405176:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004f0879c9. 00000100:00000001:1.0:1784949076.405181:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000010:2.0:1784949076.405185:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000800:00000200:0.0:1784949076.405189:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949076.405202:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:0.0:1784949076.405204:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000100:00000040:2.0:1784949076.405205:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949076.405224:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.405225:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949076.405235:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000200:0.0:1784949076.405237:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949076.405240:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.405242:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.405252:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949076.405258:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949076.405259:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949076.405261:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.405263:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:0.0:1784949076.405269:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949076.405281:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949076.405285:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949076.405288:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.405291:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.405297:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.405306:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949076.405307:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2157 usecs (3 jiffies) 00000100:00000001:0.0:1784949076.405338:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306015748 00000100:00000001:0.0:1784949076.405344:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306015787 00000100:00000001:0.0:1784949076.405348:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306015748 00000100:00000001:0.0:1784949076.405352:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306015787 00000100:00000001:0.0:1784949076.405355:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306015748 00000100:00000001:0.0:1784949076.405358:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306015787 00000100:00000001:2.0:1784949076.405358:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949076.405366:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306015748 00000100:00000001:2.0:1784949076.405366:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949076.405374:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306015787 00000100:00000001:3.0:1784949076.405376:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949076.405377:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949076.405385:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949076.405386:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 770 00000100:00000001:0.0:1784949076.405388:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.405398:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949076.405401:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949076.405401:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.405403:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949076.405414:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165126224 : -115539544425392 : ffff96eada7f3c50) 00000100:00000001:3.0:1784949076.405415:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.405420:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949076.405427:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949076.405433:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949076.405434:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ace3646f x1871651289672320/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:737/0 lens 224/0 e 0 to 0 dl 1784949087 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949076.405437:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949076.405443:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.405452:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.405462:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949076.405468:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000036da8adc. 00000100:00000001:2.0:1784949076.405472:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949076.405476:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000036da8adc. 00000100:00100000:2.0:1784949076.405482:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ace3646f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289672320:12345-0@lo:400: 00000100:00000200:2.0:1784949076.405489:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289672320 00000020:00000001:2.0:1784949076.405494:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949076.405498:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949076.405501:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949076.405504:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949076.405506:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784949076.405510:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949076.405513:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949076.405516:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949076.405518:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949076.405521:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949076.405523:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949076.405526:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949076.405529:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949076.405534:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d78419ed. 02000000:00000001:2.0:1784949076.405537:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.405540:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949076.405543:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949076.405546:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289672320 00010000:00000001:2.0:1784949076.405550:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949076.405555:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ace3646f x1871651289672320/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:737/0 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949076.405565:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949076.405568:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949076.405569:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00001000:2.0:1784949076.405572:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=98 v=5 (1 1 1 1) 00000800:00000010:3.0:1784949076.405577:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000100:00000001:2.0:1784949076.405582:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1784949076.405587:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949076.405589:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949076.405598:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:0.0:1784949076.405600:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784949076.405623:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:3.0:1784949076.405624:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:2.0:1784949076.405630:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949076.405636:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e7a7cb2 00000100:00000001:2.0:1784949076.405637:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:0.0:1784949076.405641:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949076.405644:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002e7a7cb2. 00000100:00000040:2.0:1784949076.405650:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:0.0:1784949076.405656:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784949076.405658:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000001:3.0:1784949076.405660:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:0.0:1784949076.405672:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949076.405672:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289672320, offset 224 00000100:00000200:3.0:1784949076.405674:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949076.405697:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784949076.405722:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949076.405734:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949076.405736:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949076.405742:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949076.405756:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ab00 00000400:00000200:2.0:1784949076.405759:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643aa80 00000100:00000001:3.0:1784949076.405775:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949076.405778:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 168144 00000100:00000001:3.0:1784949076.405780:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784949076.405795:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949076.405797:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8d51 [1] + 224 00000800:00000001:0.0:1784949076.405807:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949076.405809:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949076.405814:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949076.405822:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949076.405829:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949076.405831:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949076.405838:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949076.405840:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949076.405845:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949076.405856:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c1a2ac13. 00000100:00000040:0.0:1784949076.405862:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000c1a2ac13 x1871651289672448 msgsize 224 00000100:00000040:2.0:1784949076.405863:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949076.405889:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949076.405947:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949076.405956:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949076.405959:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949076.405969:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949076.405973:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000100:00000001:3.0:1784949076.405974:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:2.0:1784949076.405983:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00100000:3.0:1784949076.405984:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289672448 00000100:00000001:1.0:1784949076.405991:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949076.405992:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949076.405994:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949076.406001:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949076.406007:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949076.406008:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949076.406009:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949076.406026:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949076.406029:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949076.406032:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1784949076.406032:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289672448 00000400:00000001:1.0:1784949076.406042:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1784949076.406044:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:3.0:1784949076.406044:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949076.406058:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:1.0:1784949076.406060:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002e5b8264. 00010000:00000001:2.0:1784949076.406060:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949076.406072:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949076.406074:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784949076.406075:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000020:00000001:2.0:1784949076.406080:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:1.0:1784949076.406085:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000020:00000040:3.0:1784949076.406102:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949076.406109:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949076.406111:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1784949076.406117:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ace3646f x1871651289672320/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:737/0 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949076.406121:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784949076.406127:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949076.406139:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949076.406149:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:3.0:1784949076.406161:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949076.406176:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000020:00000010:3.0:1784949076.406196:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c73b3f00. 00000020:00000010:3.0:1784949076.406211:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b16a60b2. 00000100:00100000:2.0:1784949076.406232:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ace3646f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289672320:12345-0@lo:400: Request processed in 741us (1379us total) trans 0 rc 0/0 00000100:00000200:1.0:1784949076.406233:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949076.406241:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000040:3.0:1784949076.406260:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949076.406275:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949076.406283:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:2.0:1784949076.406291:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 770 00000100:00000001:1.0:1784949076.406294:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949076.406294:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949076.406298:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949076.406302:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949076.406307:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949076.406309:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:3.0:1784949076.406310:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949076.406314:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000001:1.0:1784949076.406317:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949076.406317:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949076.406338:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949076.406340:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949076.406344:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949076.406346:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949076.406352:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004f0879c9. 00000100:00001000:1.0:1784949076.406354:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=98 v=5 (5 5 5 5) 00000100:00000001:3.0:1784949076.406365:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784949076.406370:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=98 v=5 (1 1 1 1) 00000020:00000010:2.0:1784949076.406374:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784949076.406376:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949076.406379:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949076.406379:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000010:2.0:1784949076.406384:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e5260462. 00010000:00000001:1.0:1784949076.406389:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784949076.406391:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000040:2.0:1784949076.406398:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000001:1.0:1784949076.406405:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.406410:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949076.406414:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00100000:3.0:1784949076.406419:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 746 02000000:00000010:2.0:1784949076.406424:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d78419ed. 00000100:00000040:1.0:1784949076.406429:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949076.406431:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:2.0:1784949076.406437:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949076.406445:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554978816 : -115544154572800 : ffff96e9c7b5f800) 00000100:00000001:2.0:1784949076.406449:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949076.406469:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000c1a2ac13 x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:737/0 lens 224/0 e 0 to 0 dl 1784949087 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949076.406470:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949076.406479:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949076.406485:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002f32d921. 00000100:00000001:1.0:1784949076.406500:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:2.0:1784949076.406503:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004764d328. 00000100:00000001:1.0:1784949076.406510:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949076.406512:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949076.406526:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949076.406527:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:3.0:1784949076.406537:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1784949076.406544:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289672320:0@lo:400: 00000020:00000010:3.0:1784949076.406545:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000100:00000001:1.0:1784949076.406562:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949076.406566:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000c1a2ac13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289672448:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949076.406568:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:3.0:1784949076.406576:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289672448 00000100:00000040:1.0:1784949076.406577:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006d868603 x1871651289672320/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949076.406580:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949076.406596:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949076.406612:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1784949076.406614:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000010:1.0:1784949076.406618:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 00000020:00000001:3.0:1784949076.406618:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949076.406628:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949076.406628:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949076.406643:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:3.0:1784949076.406643:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000040:1.0:1784949076.406651:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949076.406652:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949076.406660:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:3.0:1784949076.406661:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949076.406670:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 00000020:00000001:3.0:1784949076.406672:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949076.406684:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:1.0:1784949076.406688:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949076.406702:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949076.406705:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006d868603. 00000020:00000001:3.0:1784949076.406707:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949076.406721:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949076.406727:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949076.406729:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949076.406734:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a6bc3bb1. 00000100:00000001:1.0:1784949076.406737:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1784949076.406738:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.406740:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949076.406744:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.406745:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949076.406758:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949076.406761:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949076.406769:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949076.406770:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289672448 00010000:00000001:3.0:1784949076.406787:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949076.406788:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949076.406796:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784949076.406803:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000c1a2ac13 x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:737/0 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949076.406814:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949076.406817:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949076.406820:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=93 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949076.406824:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949076.406827:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949076.406829:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949076.406832:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949076.406834:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949076.406836:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949076.406839:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949076.406842:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000100:00000200:3.0:1784949076.406846:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289672448, offset 224 00000400:00000200:3.0:1784949076.406852:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949076.406861:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949076.406866:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3578:3578:241:4294967295] 192.168.202.137@tcp LPNI seq info [2303:2303:8:4294967295] 00000400:00000200:3.0:1784949076.406877:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949076.406884:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949076.406888:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949076.406893:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949076.406899:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949076.406903:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949076.406951:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.406955:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949076.406959:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949076.406962:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949076.406964:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949076.406971:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c1a2ac13 x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:737/0 lens 224/224 e 0 to 0 dl 1784949087 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949076.406982:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000c1a2ac13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289672448:12345-192.168.202.137@tcp:400: Request processed in 471us (1118us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949076.406994:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 746 00000100:00000040:3.0:1784949076.406998:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949076.407000:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949076.407002:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949076.407005:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b16a60b2. 00000020:00000010:3.0:1784949076.407010:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949076.407014:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c73b3f00. 00000020:00000040:3.0:1784949076.407019:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949076.407022:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949076.407072:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949076.407075:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949076.407083:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784949076.407084:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000800:00000001:3.0:1784949076.407094:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949076.407096:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949076.407099:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949076.407127:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949076.407132:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949076.407137:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000400:00000010:0.0:1784949076.407143:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000100:00000001:0.0:1784949076.407149:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:3.0:1784949076.407150:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ab00 00000100:00000001:0.0:1784949076.407155:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949076.407163:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8d59 [1] + 224 00000800:00000001:3.0:1784949076.407175:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949076.407189:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949076.407192:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949076.407197:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949076.407203:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949076.407208:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949076.407219:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949076.407242:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949076.407249:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949076.407252:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949076.407289:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949076.407294:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949076.407301:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949076.407304:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949076.407307:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000400:00000200:0.0:1784949076.407313:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:0.0:1784949076.407315:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000100:00000001:0.0:1784949076.407344:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949076.407352:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949076.407383:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949076.407393:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949076.407395:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949076.407398:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949076.407401:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949076.407403:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949076.407406:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949076.407409:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949076.407415:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=93 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949076.407419:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=93 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949076.407422:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949076.407424:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949076.407427:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949076.407432:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949076.407441:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.407445:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949076.407447:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949076.407449:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949076.407450:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949076.407454:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949076.407462:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949076.407465:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949076.407467:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949076.407470:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949076.407478:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289672448:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949076.407484:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949076.407486:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949076.407489:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006e540bde x1871651289672448/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949076.407496:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949076.407498:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949076.407501:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949076.407506:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949076.407507:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949076.407509:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949076.407511:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949076.407514:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 02000000:00000001:0.0:1784949076.407517:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949076.407518:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949076.407520:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006e540bde. 00000100:00000001:0.0:1784949076.407523:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949076.407525:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949076.407528:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949076.407532:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949076.407534:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949076.407536:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949076.407539:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949076.447999:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306015787, at 4306015788 00000100:00000040:0.0:1784949076.448008:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306015787, at 4306015788 00000100:00000010:3.0:1784949076.448019:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:0.0:1784949076.448022:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:1.0:1784949076.448075:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306015787, at 4306015788 00000100:00000040:2.0:1784949076.448077:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306015787, at 4306015788 00000100:00000010:1.0:1784949076.448101:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:2.0:1784949076.448105:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949077.236020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949077.236026:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949077.236031:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949077.236033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949077.236110:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949077.236114:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949077.236117:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949077.427974:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949077.427988:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949077.427989:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949077.427994:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949077.427997:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949077.428000:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949077.428004:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949078.260004:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949078.260010:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949078.260015:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949078.260017:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949078.260099:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949078.260105:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949078.260109:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949078.451961:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949078.451977:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949078.451980:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949078.451985:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949078.451988:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949078.451992:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949078.451995:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949079.283998:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949079.284004:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949079.284009:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949079.284010:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949079.284071:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949079.284075:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949079.284078:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949079.476050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949079.476073:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949079.476075:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949079.476084:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949079.476089:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949079.476095:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949079.476099:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949080.308002:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949080.308009:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949080.308014:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949080.308016:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949080.308098:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949080.308104:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949080.308108:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949080.500060:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949080.500069:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949080.500080:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949055/real 1784949055] req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949080 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784949080.500096:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949080.500098:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949080.500101:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005e572277. 00000400:00000200:2.0:1784949080.500105:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:2.0:1784949080.500108:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000100:00000001:2.0:1784949080.500111:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949080.500114:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949080 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949080.500122:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949080 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.500127:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949080.500129:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.500131:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949080.500132:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949080.500134:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949080 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.500139:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.500145:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949080.500147:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.500150:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949080.500153:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949080.500155:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949080.500158:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949080 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.500164:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949080.500167:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.500169:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949080.500170:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.500172:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949080.500174:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949080.500177:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949080.500198:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:2.0:1784949080.500201:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949080.500204:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949080 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949080.500210:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000001445c4ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289671296:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784949080.500217:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.500220:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949080.500220:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1784949080.500224:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000001445c4ff x1871651289671296/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949080 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.500228:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949080.500235:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000010:3.0:1784949080.500235:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 02000000:00000001:2.0:1784949080.500238:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:3.0:1784949080.500241:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 02000000:00000010:2.0:1784949080.500242:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000008905542c. 00000100:00000001:3.0:1784949080.500247:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949080.500249:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1784949080.500251:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1784949080.500252:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949080.500255:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784949080.500256:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949080.500257:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949080.500262:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000007617a2e3. 02000000:00000010:3.0:1784949080.500265:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 02000000:00000001:2.0:1784949080.500266:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949080.500269:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949080.500271:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000001445c4ff. 00000100:00000001:3.0:1784949080.500273:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.500275:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949080.500278:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.500279:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949080.500282:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949080.500284:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949080.500304:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.500308:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949080.500311:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.500315:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949080.500318:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949080.500322:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_00_03+3] req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.500349:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949080.500352:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949080.500355:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784949080.500357:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784949080.500359:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784949080.500363:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949080.500363:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1784949080.500368:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1784949080.500370:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11327 00000100:00000001:0.0:1784949080.500372:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949080.500372:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949080.500377:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949080.500379:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949080.500380:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:3.0:1784949080.500388:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11326 00000100:00000001:2.0:1784949080.500395:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949080.500398:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949080.500399:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 02000000:00000001:2.0:1784949080.500403:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949080.500407:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949080.500407:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949080.500412:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949080.500413:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949080.500417:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00100000:2.0:1784949080.500419:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289672576:0@lo:400: 00000100:00000001:3.0:1784949080.500421:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:2.0:1784949080.500425:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1784949080.500426:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949080.500428:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:3.0:1784949080.500430:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1784949080.500431:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949080.500434:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949080.500434:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:2.0:1784949080.500439:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1784949080.500440:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000010:2.0:1784949080.500443:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5260462. 00000020:00000001:3.0:1784949080.500443:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:3.0:1784949080.500447:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000001:2.0:1784949080.500448:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.500452:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949080.500454:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000010:2.0:1784949080.500455:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e3d6da39. 00000100:00000001:3.0:1784949080.500457:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:3.0:1784949080.500460:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:2.0:1784949080.500461:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00080000:3.0:1784949080.500465:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000200:2.0:1784949080.500467:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289672576, portal 25 00000100:00080000:3.0:1784949080.500468:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:2.0:1784949080.500472:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949080.500473:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.500476:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 02000000:00000001:3.0:1784949080.500477:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1784949080.500482:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949080.500483:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949080.500489:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.500493:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1784949080.500494:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949080.500496:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949080.500499:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000010:3.0:1784949080.500502:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000074c559e1. 00000400:00000010:2.0:1784949080.500503:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00000200:2.0:1784949080.500508:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289672576, offset 0 00000020:00000040:3.0:1784949080.500508:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784949080.500514:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784949080.500516:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949080.500519:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949080.500521:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949080.500524:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949080.500525:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949080.500528:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000000239f355. 00000400:00000200:2.0:1784949080.500532:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949080.500533:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949080.500540:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949080.500546:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ab80 00000100:00000001:3.0:1784949080.500553:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949080.500555:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949080.500558:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x83e5 [2] + 0 00000100:00000040:3.0:1784949080.500562:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000074c559e1] to pc [ptlrpcd_rcv+-1] req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949080.500564:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ea556a29. 00000400:00000200:2.0:1784949080.500571:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000007c7d797b 00000400:00000200:2.0:1784949080.500579:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949080.500582:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949080.500586:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:3.0:1784949080.500587:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949080.500593:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:2.0:1784949080.500594:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c7d797b 00000100:00000010:3.0:1784949080.500595:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003492edb3. 00000400:00000010:2.0:1784949080.500597:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000007c7d797b. 00000020:00000040:3.0:1784949080.500599:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949080.500603:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949080.500604:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1784949080.500606:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:3.0:1784949080.500607:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949080.500610:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949080.500612:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000006f58d1a4 x1871651289672576 msgsize 224 02000000:00000010:3.0:1784949080.500614:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:0.0:1784949080.500621:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949080.500622:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949080.500626:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:0.0:1784949080.500628:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784949080.500629:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:3.0:1784949080.500632:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949080.500634:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949080.500640:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949080.500641:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1784949080.500645:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.500645:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949080.500648:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:2.0:1784949080.500653:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000100:00000040:3.0:1784949080.500655:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003492edb3] to pc [ptlrpcd_00_00+0] req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.500658:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949080.500667:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949080.500672:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949080.500683:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289672576 00000100:00000001:0.0:1784949080.500685:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949080.500691:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949080.500693:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949080.500695:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949080.500702:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949080.500703:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1784949080.500706:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949080.500713:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949080.500714:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949080.500721:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.500721:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949080.500726:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949080.500727:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289672704:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949080.500727:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949080.500735:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949080.500735:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306019882 00000100:00000040:3.0:1784949080.500735:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11357) 00000100:00000001:0.0:1784949080.500739:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949080.500742:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949080.500744:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f4af3cce. 00000100:00000001:0.0:1784949080.500745:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949080.500750:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949080.500753:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949080.500755:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.500755:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784949080.500760:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949080.500763:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949080.500764:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949080.500769:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.500771:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306019882 00000100:00000001:3.0:1784949080.500772:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949080.500774:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.500777:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949080.500786:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.500793:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949080.500795:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949080.500800:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949080.500804:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949080.500807:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949080.500810:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289672832:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949080.500814:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949080.500814:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949080.500816:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949080.500821:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949080.500822:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949080.500823:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949080.500824:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949080.500830:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e7299d92. 02000000:00000001:1.0:1784949080.500831:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949080.500834:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949080.500838:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289672576 00000400:00000010:3.0:1784949080.500839:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009ef01e88. 00000020:00000001:1.0:1784949080.500845:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949080.500847:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000010:3.0:1784949080.500848:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000020:00000001:1.0:1784949080.500854:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:3.0:1784949080.500856:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289672832, portal 10 00000020:00000040:1.0:1784949080.500859:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949080.500860:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949080.500866:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554958848 : -115544154592768 : ffff96e9c7b5aa00) 00000020:00000001:1.0:1784949080.500871:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:2.0:1784949080.500871:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000020:00000001:1.0:1784949080.500883:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:3.0:1784949080.500884:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.500885:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949080.500896:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306019882 00000100:00000001:1.0:1784949080.500899:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949080.500937:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000010:2.0:1784949080.500941:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000100:00000001:1.0:1784949080.500944:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949080.500950:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949080.500954:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:1.0:1784949080.500959:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:2.0:1784949080.500963:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306019883 00000100:00000040:3.0:1784949080.500970:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000010:2.0:1784949080.500971:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f68fd7be. 00000400:00000010:3.0:1784949080.500975:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000020:00000010:1.0:1784949080.500976:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000b241612. 00000100:00000001:2.0:1784949080.500977:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000200:3.0:1784949080.500984:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289672832, offset 0 00000100:00000001:2.0:1784949080.500985:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949080.500986:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000004db9864. 00000100:00000001:2.0:1784949080.500995:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949080.500999:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000400:00000200:3.0:1784949080.501000:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949080.501001:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.501007:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.501015:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949080.501017:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:1.0:1784949080.501020:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949080.501021:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949080.501021:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784949080.501029:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949080.501031:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949080.501034:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949080.501034:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949080.501035:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949080.501044:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.501045:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949080.501050:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949080.501062:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949080.501066:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949080.501067:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949080.501074:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3579:3579:241:4294967295] 192.168.202.137@tcp LPNI seq info [2304:2304:8:4294967295] 00000100:00000001:1.0:1784949080.501078:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949080.501096:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949080.501100:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949080.501105:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:3.0:1784949080.501105:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949080.501108:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949080.501113:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000010:3.0:1784949080.501114:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000100:00100000:1.0:1784949080.501122:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 771 00000100:00000040:1.0:1784949080.501127:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949080.501128:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949080.501134:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949080.501141:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165114960 : -115539544436656 : ffff96eada7f1050) 00000100:00000001:2.0:1784949080.501141:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949080.501154:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949080.501160:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:1.0:1784949080.501161:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006f58d1a4 x1871651289672576/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:741/0 lens 224/0 e 0 to 0 dl 1784949091 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949080.501175:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949080.501177:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949080.501178:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:1.0:1784949080.501186:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949080.501187:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2160 usecs (3 jiffies) 00000020:00000010:1.0:1784949080.501199:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:3.0:1784949080.501199:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306019844 00000100:00000001:3.0:1784949080.501210:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306019882 00000100:00100000:1.0:1784949080.501216:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006f58d1a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289672576:12345-0@lo:400: 00000100:00000001:3.0:1784949080.501217:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306019844 00000100:00000001:3.0:1784949080.501230:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306019882 00000100:00000200:1.0:1784949080.501231:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289672576 00000100:00000001:3.0:1784949080.501244:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306019844 00000020:00000001:1.0:1784949080.501245:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949080.501253:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306019882 00000020:00000001:1.0:1784949080.501255:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949080.501257:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306019844 00000020:00000001:1.0:1784949080.501259:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.501260:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306019883 00000020:00000001:1.0:1784949080.501272:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784949080.501273:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949080.501280:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:2.0:1784949080.501281:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949080.501282:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.501303:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949080.501304:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000800:00000001:0.0:1784949080.501305:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949080.501305:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949080.501307:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949080.501312:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949080.501314:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784949080.501316:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949080.501317:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1784949080.501319:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949080.501321:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000001:0.0:1784949080.501323:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949080.501327:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949080.501328:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949080.501329:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949080.501330:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949080.501342:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:2.0:1784949080.501343:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000100:00000001:3.0:1784949080.501343:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:0.0:1784949080.501344:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949080.501353:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000100:00000001:1.0:1784949080.501354:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949080.501354:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949080.501362:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949080.501364:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949080.501365:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949080.501368:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000010:1.0:1784949080.501371:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000cc641d8a. 00000100:00000200:2.0:1784949080.501374:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949080.501378:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949080.501387:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949080.501388:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ac80 00000020:00000001:1.0:1784949080.501399:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949080.501412:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 168368 00010000:00000040:1.0:1784949080.501412:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289672576 00000100:00000001:2.0:1784949080.501414:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1784949080.501422:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:2.0:1784949080.501423:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784949080.501432:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006f58d1a4 x1871651289672576/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:741/0 lens 224/224 e 0 to 0 dl 1784949091 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.501432:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949080.501433:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949080.501445:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949080.501445:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949080.501457:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1784949080.501463:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949080.501468:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949080.501470:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00001000:1.0:1784949080.501476:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=102 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949080.501478:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949080.501483:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784949080.501483:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949080.501491:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000010:0.0:1784949080.501492:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000df8ddd96. 00000100:00000001:1.0:1784949080.501500:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000040:0.0:1784949080.501501:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000df8ddd96 x1871651289672832 msgsize 224 02000000:00000001:1.0:1784949080.501507:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949080.501511:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949080.501519:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949080.501519:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949080.501527:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949080.501530:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00000001:0.0:1784949080.501533:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:1.0:1784949080.501536:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289672576, offset 224 00000800:00000001:0.0:1784949080.501543:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949080.501544:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:0.0:1784949080.501548:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949080.501557:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949080.501567:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784949080.501570:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ab80 00000100:00100000:2.0:1784949080.501572:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289672832 02000000:00000001:2.0:1784949080.501576:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949080.501578:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784949080.501583:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8d6d [1] + 224 00000100:00000001:2.0:1784949080.501584:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949080.501598:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949080.501600:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:2.0:1784949080.501602:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289672832 00000020:00000001:2.0:1784949080.501605:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1784949080.501609:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000040:2.0:1784949080.501610:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949080.501614:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949080.501615:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1784949080.501617:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:1.0:1784949080.501622:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949080.501623:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949080.501633:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949080.501645:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949080.501649:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000036da8adc. 00000100:00000040:1.0:1784949080.501650:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.501657:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949080.501669:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000036da8adc. 00000020:00000010:2.0:1784949080.501679:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007a6587e5. 00000020:00000010:2.0:1784949080.501684:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004f0879c9. 00000100:00000001:1.0:1784949080.501689:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784949080.501690:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000200:1.0:1784949080.501698:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000100:00000040:2.0:1784949080.501701:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:1.0:1784949080.501702:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784949080.501711:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949080.501711:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949080.501720:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949080.501721:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949080.501732:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949080.501733:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949080.501749:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784949080.501752:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:2.0:1784949080.501752:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949080.501755:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949080.501759:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.501760:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949080.501761:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000001:1.0:1784949080.501766:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784949080.501767:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949080.501772:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006f58d1a4 x1871651289672576/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:741/0 lens 224/224 e 0 to 0 dl 1784949091 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784949080.501773:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949080.501776:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784949080.501780:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949080.501783:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e3d6da39. 00000400:00000200:3.0:1784949080.501797:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000100:00000001:2.0:1784949080.501801:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:1.0:1784949080.501802:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006f58d1a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289672576:12345-0@lo:400: Request processed in 623us (1185us total) trans 0 rc 0/0 00000400:00000010:3.0:1784949080.501804:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:2.0:1784949080.501806:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949080.501815:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949080.501819:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 747 00000100:00100000:1.0:1784949080.501827:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 771 00000100:00000001:0.0:1784949080.501828:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000200:3.0:1784949080.501828:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949080.501829:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949080.501840:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:1.0:1784949080.501843:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949080.501843:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554970752 : -115544154580864 : ffff96e9c7b5d880) 00000100:00000001:0.0:1784949080.501846:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949080.501856:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949080.501859:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949080.501862:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:2.0:1784949080.501862:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000df8ddd96 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:741/0 lens 224/0 e 0 to 0 dl 1784949091 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949080.501875:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000004db9864. 00000100:00000200:3.0:1784949080.501877:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.501887:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949080.501888:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949080.501890:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784949080.501893:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000036da8adc. 00000100:00000001:3.0:1784949080.501899:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949080.501900:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000b241612. 00000100:00000001:2.0:1784949080.501901:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949080.501948:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:2.0:1784949080.501951:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000036da8adc. 02000000:00000001:3.0:1784949080.501953:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:1.0:1784949080.501954:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949080.501958:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949080.501962:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949080.501963:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000df8ddd96 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289672832:12345-192.168.202.137@tcp:400: 02000000:00000001:1.0:1784949080.501964:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:3.0:1784949080.501969:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949080.501970:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000cc641d8a. 00000100:00000200:2.0:1784949080.501971:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289672832 00000020:00000001:2.0:1784949080.501983:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:1.0:1784949080.501984:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:3.0:1784949080.501985:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=102 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949080.501992:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=102 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949080.501993:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949080.501994:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949080.502002:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949080.502009:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949080.502009:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:2.0:1784949080.502010:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949080.502018:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:3.0:1784949080.502018:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949080.502019:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e5b8264. 00010000:00000001:3.0:1784949080.502030:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:2.0:1784949080.502031:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:1.0:1784949080.502032:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d51a2e44. 00010000:00000001:3.0:1784949080.502034:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949080.502036:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949080.502043:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949080.502044:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949080.502047:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949080.502052:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949080.502053:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949080.502056:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949080.502062:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949080.502064:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949080.502065:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1784949080.502068:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784949080.502068:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784949080.502073:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000860704e4. 00000100:00000040:3.0:1784949080.502073:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949080.502078:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949080.502082:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949080.502086:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949080.502089:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289672576:0@lo:400: 00010000:00000040:2.0:1784949080.502090:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289672832 00010000:00000001:2.0:1784949080.502095:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949080.502098:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:2.0:1784949080.502101:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000df8ddd96 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:741/0 lens 224/224 e 0 to 0 dl 1784949091 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.502102:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949080.502106:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289672576/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949080.502122:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949080.502123:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:2.0:1784949080.502125:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949080.502126:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00001000:2.0:1784949080.502130:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=97 v=5 (1 1 1 1) 02000000:00000010:3.0:1784949080.502130:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5260462. 00000100:00000001:2.0:1784949080.502136:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784949080.502137:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:2.0:1784949080.502140:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000020:00000001:3.0:1784949080.502141:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949080.502144:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949080.502145:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949080.502149:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:3.0:1784949080.502149:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949080.502152:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:2.0:1784949080.502153:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949080.502156:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1784949080.502157:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949080.502159:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1784949080.502163:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000010:3.0:1784949080.502163:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000400:00000010:2.0:1784949080.502166:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00000200:2.0:1784949080.502170:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289672832, offset 224 00000400:00000200:2.0:1784949080.502173:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949080.502176:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949080.502178:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949080.502182:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949080.502183:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949080.502190:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1784949080.502191:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3580:3580:241:4294967295] 192.168.202.137@tcp LPNI seq info [2305:2305:8:4294967295] 00000100:00000001:3.0:1784949080.502193:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.502198:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949080.502201:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949080.502206:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949080.502211:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949080.502214:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005523afda. 00000800:00000200:2.0:1784949080.502217:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949080.502221:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949080.502223:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005523afda type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949080.502231:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949080.502234:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949080.502236:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949080.502237:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949080.502239:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949080.502242:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000df8ddd96 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:741/0 lens 224/224 e 0 to 0 dl 1784949091 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949080.502248:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000df8ddd96 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289672832:12345-192.168.202.137@tcp:400: Request processed in 361us (745us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949080.502252:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 747 00000100:00000040:2.0:1784949080.502255:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949080.502257:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949080.502259:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949080.502262:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004f0879c9. 00000020:00000010:2.0:1784949080.502265:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949080.502267:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007a6587e5. 00000020:00000040:2.0:1784949080.502272:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949080.502274:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949080.502319:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949080.502322:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949080.502325:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949080.502325:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005523afda. 00000800:00000001:2.0:1784949080.502331:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949080.502333:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949080.502333:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949080.502344:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949080.502346:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949080.502349:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:3.0:1784949080.502351:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000400:00000200:2.0:1784949080.502355:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ac80 00000100:00000001:3.0:1784949080.502355:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949080.502358:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949080.502363:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8d75 [1] + 224 00000800:00000001:2.0:1784949080.502367:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949080.502376:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949080.502378:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949080.502380:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949080.502383:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949080.502385:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949080.502391:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949080.502403:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949080.502406:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949080.502407:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949080.502428:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949080.502430:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949080.502432:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949080.502433:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949080.502434:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009ef01e88. 00000400:00000200:3.0:1784949080.502437:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949080.502438:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:3.0:1784949080.502440:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949080.502443:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949080.502449:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.502454:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949080.502455:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949080.502456:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949080.502457:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949080.502457:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949080.502458:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949080.502460:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949080.502462:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=97 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949080.502464:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=97 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949080.502466:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949080.502466:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949080.502468:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949080.502470:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949080.502474:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.502475:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949080.502476:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949080.502478:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949080.502478:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949080.502480:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.502484:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949080.502485:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949080.502486:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949080.502488:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949080.502492:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289672832:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949080.502495:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949080.502496:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949080.502497:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003492edb3 x1871651289672832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949096 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949080.502501:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949080.502502:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949080.502503:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e7299d92. 02000000:00000001:3.0:1784949080.502505:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949080.502506:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949080.502507:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949080.502508:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949080.502509:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:3.0:1784949080.502510:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949080.502511:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949080.502512:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003492edb3. 00000100:00000001:3.0:1784949080.502514:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949080.502514:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949080.502515:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949080.502517:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949080.502518:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949080.502519:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949080.502520:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949080.542979:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306019882, at 4306019883 00000100:00000040:1.0:1784949080.542979:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306019882, at 4306019883 00000100:00000010:1.0:1784949080.542995:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000010:0.0:1784949080.542999:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f4af3cce. 00000100:00000040:2.0:1784949080.543968:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306019882, at 4306019884 00000100:00000040:3.0:1784949080.543973:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306019883, at 4306019884 00000100:00000010:2.0:1784949080.543976:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000100:00000010:3.0:1784949080.543982:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f68fd7be. 00010000:00000001:2.0:1784949081.332018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949081.332027:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949081.332034:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949081.332038:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949081.332042:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949081.332045:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949081.332049:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949081.332051:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949081.332054:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949081.332056:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949081.332195:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949081.332200:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949081.332204:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949081.523997:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949081.524005:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949081.524021:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949080/real 0] req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949081 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.524049:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.524053:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949081.524056:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949081.524061:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949081 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.524072:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.524082:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949081.524086:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949081.524090:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949081.524093:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949081.524097:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949081.524104:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949081 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.524114:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949081.524118:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.524120:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949081.524121:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.524124:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949081.524126:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949081.524131:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949081.524168:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784949081.524172:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949081.524178:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949081 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949081.524188:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289672704:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949081.524196:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.524200:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949081.524203:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000074c559e1 x1871651289672704/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949081 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.524220:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784949081.524224:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949081.524227:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784949081.524231:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949081.524237:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000000239f355. 02000000:00000001:0.0:1784949081.524245:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949081.524250:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949081.524253:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000074c559e1. 00000100:00000001:0.0:1784949081.524263:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949081.524266:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949081.524274:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949081.524275:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.524310:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949081.524324:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949081.524337:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949081.524344:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949081.524345:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949081.524363:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb4f6eba. 00000020:00000040:1.0:1784949081.524374:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949081.524381:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949081.524386:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949081.524394:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949081.524405:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c158c1d0. 00000100:00000001:1.0:1784949081.524417:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949081.524428:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949081.524441:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949081.524444:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949081.524452:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb4f6eba] to pc [ptlrpcd_00_01+1] req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949081.524491:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949081.524494:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949081.524497:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784949081.524501:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784949081.524505:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784949081.524509:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784949081.524516:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11327 00000100:00000001:0.0:1784949081.524528:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784949081.524530:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:0.0:1784949081.524535:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1784949081.524540:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11352 00000100:00000001:0.0:1784949081.524541:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949081.524548:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949081.524559:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949081.524565:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949081.524565:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00080000:1.0:1784949081.524581:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1784949081.524600:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949081.524608:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949081.524609:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949081.524627:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949081.524631:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784949081.524634:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949081.524636:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949081.524641:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:0.0:1784949081.524642:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1784949081.524650:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:0.0:1784949081.524651:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784949081.524661:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:0.0:1784949081.524666:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949081.524667:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:1.0:1784949081.524680:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00100000:0.0:1784949081.524681:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289672960:0@lo:400: 00000100:00000001:0.0:1784949081.524692:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949081.524693:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949081.524698:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949081.524700:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949081.524701:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949081.524702:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949081.524709:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949081.524709:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:0.0:1784949081.524726:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1784949081.524734:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 02000000:00000010:0.0:1784949081.524739:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 00000100:00080000:1.0:1784949081.524740:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 02000000:00000001:0.0:1784949081.524751:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.524752:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949081.524769:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1784949081.524776:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 02000000:00000001:1.0:1784949081.524780:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949081.524786:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949081.524788:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00000001:1.0:1784949081.524789:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1784949081.524801:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949081.524807:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289672960, portal 25 00000100:00000001:0.0:1784949081.524812:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949081.524815:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549148160 : -115544160403456 : ffff96e9c75d0000) 00000100:00000040:0.0:1784949081.524822:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.524833:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949081.524837:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949081.524842:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000100:00000010:1.0:1784949081.524845:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006a448376. 00000100:00000200:0.0:1784949081.524847:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289672960, offset 0 00000020:00000040:1.0:1784949081.524854:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:1.0:1784949081.524862:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1784949081.524869:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949081.524871:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949081.524881:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949081.524885:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949081.524889:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949081.524895:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000cc641d8a. 00000400:00000200:0.0:1784949081.524955:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949081.524963:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949081.524981:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784949081.524984:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ad00 00000100:00000001:1.0:1784949081.525007:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1784949081.525008:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8401 [2] + 0 00000100:00000001:1.0:1784949081.525014:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949081.525020:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b2f60a65. 00000100:00000040:1.0:1784949081.525024:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a448376] to pc [ptlrpcd_rcv+-1] req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949081.525028:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000026a1c0e3 00000400:00000200:0.0:1784949081.525044:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949081.525060:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949081.525061:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:1.0:1784949081.525076:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1784949081.525080:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026a1c0e3 00000100:00000001:1.0:1784949081.525083:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000010:0.0:1784949081.525086:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000026a1c0e3. 00000100:00000010:1.0:1784949081.525090:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001dee7260. 00000100:00000001:0.0:1784949081.525097:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000020:00000040:1.0:1784949081.525097:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000200:0.0:1784949081.525111:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784949081.525112:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949081.525124:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1784949081.525126:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000e78df062 x1871651289672960 msgsize 224 02000000:00000001:1.0:1784949081.525130:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949081.525140:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000008d5370ec. 00000100:00000001:2.0:1784949081.525140:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949081.525144:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949081.525161:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949081.525163:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949081.525163:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949081.525177:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949081.525185:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784949081.525195:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.525197:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949081.525216:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000100:00000001:3.0:1784949081.525225:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:0.0:1784949081.525229:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000100:00100000:3.0:1784949081.525240:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289672960 00000100:00000001:0.0:1784949081.525247:0:337120:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784949081.525255:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000200:0.0:1784949081.525267:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949081.525269:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949081.525269:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949081.525275:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949081.525292:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949081.525299:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001dee7260] to pc [ptlrpcd_00_02+2] req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949081.525308:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949081.525311:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949081.525330:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949081.525330:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949081.525340:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784949081.525348:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289672960 00000020:00000001:3.0:1784949081.525365:0:342757:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1784949081.525366:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949081.525377:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784949081.525389:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:2.0:1784949081.525390:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006a448376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289673088:192.168.202.137@tcp:8: 00000020:00000040:3.0:1784949081.525400:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949081.525406:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1784949081.525420:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:0.0:1784949081.525422:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949081.525425:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1784949081.525432:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:2.0:1784949081.525436:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949081.525443:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.525446:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949081.525448:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:2.0:1784949081.525455:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:3.0:1784949081.525461:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784949081.525463:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949081.525467:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11358) 02000000:00000010:2.0:1784949081.525469:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d78419ed. 00000100:00000001:3.0:1784949081.525473:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949081.525475:0:337120:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784949081.525490:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949081.525491:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784949081.525493:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.525504:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:3.0:1784949081.525505:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e7299d92. 00000100:00000001:0.0:1784949081.525506:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306020907 00000400:00000010:2.0:1784949081.525510:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000930107ef. 00000100:00000001:1.0:1784949081.525512:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1784949081.525520:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000974f7da. 00000100:00000001:1.0:1784949081.525521:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:0.0:1784949081.525524:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000400:00000010:2.0:1784949081.525528:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000020:00000010:3.0:1784949081.525536:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000040:1.0:1784949081.525538:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.525540:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000200:2.0:1784949081.525543:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289673088, portal 4 00000100:00000001:2.0:1784949081.525557:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949081.525561:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306020907 00000100:00000001:2.0:1784949081.525562:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637802880 : -115544071748736 : ffff96e9cca5c380) 00000100:00000001:1.0:1784949081.525568:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1784949081.525568:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000010:0.0:1784949081.525576:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:1.0:1784949081.525580:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949081.525583:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:2.0:1784949081.525585:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949106 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.525590:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949081.525594:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:1.0:1784949081.525598:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949081.525607:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949081.525609:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949081.525611:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306020907 00000100:00000001:3.0:1784949081.525630:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949081.525633:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 02000000:00000001:1.0:1784949081.525633:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949081.525639:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949081.525655:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949081.525657:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949081.525658:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949081.525664:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000001dee7260 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289673216:192.168.202.137@tcp:400: 00000400:00000010:2.0:1784949081.525670:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000100:00000001:0.0:1784949081.525673:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306020907 00000100:00000001:1.0:1784949081.525682:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949081.525684:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000200:2.0:1784949081.525685:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289673088, offset 0 00000100:00000010:0.0:1784949081.525688:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:3.0:1784949081.525696:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:1.0:1784949081.525698:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949081.525701:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:1.0:1784949081.525712:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949081.525713:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949081.525718:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949081.525725:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1784949081.525727:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 772 00000100:00000001:0.0:1784949081.525736:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949081.525739:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 00000100:00000040:3.0:1784949081.525741:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:2.0:1784949081.525743:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949081.525748:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949081.525758:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949081.525760:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782156880 : -115538927394736 : ffff96eaff466050) 00000100:00000001:0.0:1784949081.525762:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949081.525772:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000010:1.0:1784949081.525777:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000010aeeaab. 00000100:00000001:0.0:1784949081.525779:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784949081.525784:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e78df062 x1871651289672960/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:742/0 lens 224/0 e 0 to 0 dl 1784949092 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949081.525788:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949081.525790:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000010:1.0:1784949081.525796:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000200:2.0:1784949081.525804:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949081.525806:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949081.525812:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:1.0:1784949081.525814:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289673216, portal 10 00000100:00000001:0.0:1784949081.525821:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949081.525822:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949081.525830:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949081.525833:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.525838:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637811840 : -115544071739776 : ffff96e9cca5e680) 00000400:00000200:2.0:1784949081.525849:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3581:3581:241:4294967295] 192.168.202.137@tcp LPNI seq info [2306:2306:8:4294967295] 00000100:00000001:3.0:1784949081.525860:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949081.525868:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949081.525870:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784949081.525882:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949081.525883:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949081.525894:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949081.525903:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:3.0:1784949081.525945:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00000001:1.0:1784949081.525963:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1784949081.525973:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e78df062 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289672960:12345-0@lo:400: 00000100:00000040:1.0:1784949081.525996:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000800:00000200:2.0:1784949081.525998:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000200:3.0:1784949081.525998:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289672960 00000400:00000010:1.0:1784949081.526016:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000020:00000001:3.0:1784949081.526017:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000010:2.0:1784949081.526020:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fbc83221. 00000100:00000200:1.0:1784949081.526030:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289673216, offset 0 00000020:00000001:3.0:1784949081.526031:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949081.526041:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949081.526046:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:3.0:1784949081.526050:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:1.0:1784949081.526058:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784949081.526058:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:2.0:1784949081.526072:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:3.0:1784949081.526074:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949081.526079:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949081.526082:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fbc83221 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1784949081.526085:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:3.0:1784949081.526086:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949081.526094:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949081.526106:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949081.526110:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784949081.526117:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949081.526122:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949081.526123:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784949081.526123:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949081.526136:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784949081.526140:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000400:00000200:1.0:1784949081.526144:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1784949081.526146:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d02b883a. 00000100:00000001:2.0:1784949081.526155:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306020908 02000000:00000001:3.0:1784949081.526156:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949081.526158:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949081.526174:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949081.526178:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4306020907 00000400:00000200:1.0:1784949081.526183:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3582:3582:240:4294967295] 192.168.202.137@tcp LPNI seq info [2307:2307:7:4294967295] 00000020:00000001:3.0:1784949081.526185:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949081.526198:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306020908 00010000:00000040:3.0:1784949081.526204:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289672960 00010000:00000001:3.0:1784949081.526213:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949081.526214:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4306020907 00000100:00000001:2.0:1784949081.526233:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306020908 00010000:00000200:3.0:1784949081.526236:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e78df062 x1871651289672960/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:742/0 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949081.526242:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784949081.526251:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4306020907 00000800:00000200:0.0:1784949081.526259:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949081.526275:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306020908 00000800:00000200:1.0:1784949081.526293:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949081.526301:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fbc83221. 00000100:00000001:2.0:1784949081.526311:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4306020907 00000800:00000010:1.0:1784949081.526317:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002ddfcc2d. 00010000:00000001:3.0:1784949081.526318:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:0.0:1784949081.526320:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949081.526322:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784949081.526331:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949081.526337:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949081.526339:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=103 v=5 (1 1 1 1) 00000400:00000200:0.0:1784949081.526348:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000200:1.0:1784949081.526351:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949081.526354:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949081.526373:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949081.526374:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1784949081.526375:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000100:00000001:2.0:1784949081.526383:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:1.0:1784949081.526387:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000010:0.0:1784949081.526389:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000100:00000040:3.0:1784949081.526391:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949081.526392:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1784949081.526406:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949081.526412:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000200:1.0:1784949081.526414:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002ddfcc2d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949081.526414:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949081.526419:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949106 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949081.526427:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949081.526430:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784949081.526441:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949081.526441:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949081.526452:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949081.526456:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.526457:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949081.526458:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949081.526461:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949106 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949081.526480:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:1.0:1784949081.526482:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2160 usecs (3 jiffies) 00000400:00000010:3.0:1784949081.526495:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000001:1.0:1784949081.526500:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306020869 00000100:00000001:0.0:1784949081.526502:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949081.526507:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289672960, offset 224 00000800:00000001:2.0:1784949081.526513:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949081.526523:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306020908 00000100:00000001:0.0:1784949081.526524:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1784949081.526532:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949081.526536:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949081.526545:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306020869 00000800:00000001:2.0:1784949081.526549:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949081.526552:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306020908 00000100:00000001:1.0:1784949081.526558:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306020869 00000400:00000200:3.0:1784949081.526570:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949081.526571:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306020908 00000100:00000001:1.0:1784949081.526586:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306020869 00000100:00000001:1.0:1784949081.526592:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306020908 00000100:00000001:1.0:1784949081.526595:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949081.526598:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949081.526600:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ad00 00000100:00000001:1.0:1784949081.526610:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.526618:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949081.526618:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8d89 [1] + 224 00000400:00000200:3.0:1784949081.526640:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949081.526643:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949081.526647:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000001:2.0:1784949081.526652:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949081.526653:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000200:0.0:1784949081.526660:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949081.526663:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949081.526667:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:2.0:1784949081.526671:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784949081.526675:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002ddfcc2d. 00000100:00000001:1.0:1784949081.526677:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:2.0:1784949081.526678:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784949081.526679:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949081.526689:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949081.526694:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949081.526695:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949081.526708:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949081.526731:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949081.526737:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949081.526746:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:0.0:1784949081.526753:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000040:3.0:1784949081.526760:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.526761:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:1.0:1784949081.526768:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949081.526770:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ae00 00000100:00000200:0.0:1784949081.526778:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949081.526806:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 168592 00000400:00000200:1.0:1784949081.526812:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000800:00000001:2.0:1784949081.526845:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949081.526862:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643ad80 00000800:00000001:2.0:1784949081.526865:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.526870:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949081.526874:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1784949081.526886:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949081.526887:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949081.526897:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949081.526938:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:3.0:1784949081.526946:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000100:1.0:1784949081.526947:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289673088 offset 0 length 520: 4 00000100:00000001:3.0:1784949081.526955:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949081.526968:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949081.526969:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949081.526973:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 0000000072893f8d is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000040:3.0:1784949081.526989:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:1.0:1784949081.526993:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:2.0:1784949081.526998:0:337112:0:(events.c:286:request_in_callback()) Process entered 00010000:00000001:3.0:1784949081.527001:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:2.0:1784949081.527009:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:3.0:1784949081.527016:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949081.527018:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949081.527019:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000077fe3ed0. 00000020:00000001:3.0:1784949081.527027:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949081.527031:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784949081.527031:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949081.527034:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000077fe3ed0 x1871651289673216 msgsize 224 00000100:00000040:3.0:1784949081.527040:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e78df062 x1871651289672960/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:742/0 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949081.527063:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.527069:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949081.527071:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949081.527082:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:1.0:1784949081.527084:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:3.0:1784949081.527095:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e78df062 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289672960:12345-0@lo:400: Request processed in 1223us (1954us total) trans 0 rc 0/0 00000800:00000001:1.0:1784949081.527104:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:0.0:1784949081.527106:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784949081.527106:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00100000:3.0:1784949081.527117:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 772 00000400:00000001:0.0:1784949081.527122:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1784949081.527127:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000010:0.0:1784949081.527128:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000100:00000001:3.0:1784949081.527139:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000200:0.0:1784949081.527140:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000100:00000001:3.0:1784949081.527144:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000010:0.0:1784949081.527146:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000020:00000010:3.0:1784949081.527153:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000974f7da. 00000100:00000001:0.0:1784949081.527156:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949081.527169:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000200:0.0:1784949081.527170:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949081.527170:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:2.0:1784949081.527171:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:1.0:1784949081.527186:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289673216 00000800:00000001:2.0:1784949081.527190:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949081.527190:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e7299d92. 02000000:00000001:1.0:1784949081.527196:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949081.527208:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:3.0:1784949081.527211:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949081.527224:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949081.527230:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949081.527232:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:1.0:1784949081.527233:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949081.527243:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d02b883a. 00000100:00100000:1.0:1784949081.527250:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289673216 02000000:00000001:3.0:1784949081.527254:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:1.0:1784949081.527266:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949081.527268:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784949081.527295:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784949081.527298:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1784949081.527301:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949081.527303:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784949081.527307:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:3.0:1784949081.527310:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784949081.527311:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:1.0:1784949081.527312:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949081.527325:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:0.0:1784949081.527327:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:3.0:1784949081.527328:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009ef01e88. 00000100:00000001:0.0:1784949081.527337:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949081.527350:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.527353:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:3.0:1784949081.527353:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d1181415. 02000000:00000001:0.0:1784949081.527355:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949081.527363:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004204c760. 00000100:00001000:0.0:1784949081.527367:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=103 v=5 (5 5 5 5) 00000100:00000001:1.0:1784949081.527370:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00001000:0.0:1784949081.527379:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=103 v=5 (1 1 1 1) 00000020:00000010:1.0:1784949081.527380:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004204c760. 00000100:00000001:0.0:1784949081.527397:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:1.0:1784949081.527401:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ee879bbd. 00000100:00000001:0.0:1784949081.527402:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949081.527420:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:1.0:1784949081.527422:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000004db9864. 00010000:00000001:0.0:1784949081.527425:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949081.527438:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949081.527438:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:0.0:1784949081.527465:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949081.527468:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949081.527483:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949081.527494:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949081.527495:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949081.527501:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.527502:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949081.527509:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949081.527516:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949081.527517:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949081.527526:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949081.527533:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949081.527550:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289672960:0@lo:400: 00000100:00000001:1.0:1784949081.527559:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.527561:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.527567:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949081.527571:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb4f6eba x1871651289672960/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949081.527577:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949081.527581:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1784949081.527583:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:0.0:1784949081.527586:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949081.527590:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 00000100:00100000:1.0:1784949081.527600:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 748 02000000:00000001:0.0:1784949081.527603:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949081.527611:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000040:1.0:1784949081.527611:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000020:00000040:0.0:1784949081.527619:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949081.527621:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562012288 : -115544147539328 : ffff96e9c8214a80) 00000020:00000001:0.0:1784949081.527629:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949081.527632:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c158c1d0. 00000100:00000040:1.0:1784949081.527636:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000077fe3ed0 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:742/0 lens 224/0 e 0 to 0 dl 1784949092 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949081.527644:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949081.527654:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949081.527662:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb4f6eba. 00000100:00000001:1.0:1784949081.527668:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949081.527672:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000010:1.0:1784949081.527672:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004204c760. 00000100:00000001:0.0:1784949081.527680:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949081.527682:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949081.527688:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949081.527690:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004204c760. 00000100:00000001:0.0:1784949081.527696:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949081.527701:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949081.527703:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000077fe3ed0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289673216:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949081.527712:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784949081.527714:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289673216 00000100:00000001:0.0:1784949081.527716:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949081.527721:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949081.527726:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949081.527728:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949081.527731:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949081.527733:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949081.527737:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949081.527741:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949081.527744:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949081.527746:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949081.527748:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949081.527750:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949081.527755:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949081.527757:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949081.527761:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000081cf9734. 02000000:00000001:1.0:1784949081.527764:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.527766:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949081.527767:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949081.527770:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289673216 00010000:00000001:1.0:1784949081.527773:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949081.527777:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000077fe3ed0 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:742/0 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949081.527784:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949081.527787:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949081.527790:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=98 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949081.527794:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949081.527795:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949081.527797:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949081.527800:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949081.527801:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949081.527805:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949081.527807:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949081.527809:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000100:00000200:1.0:1784949081.527812:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289673216, offset 224 00000400:00000200:1.0:1784949081.527816:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949081.527823:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949081.527827:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3583:3583:241:4294967295] 192.168.202.137@tcp LPNI seq info [2308:2308:8:4294967295] 00000400:00000200:1.0:1784949081.527836:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949081.527841:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949081.527843:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000bc928ba2. 00000800:00000200:1.0:1784949081.527847:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949081.527851:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949081.527854:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000bc928ba2 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949081.527868:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949081.527870:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949081.527873:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949081.527874:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949081.527876:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949081.527878:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000077fe3ed0 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:742/0 lens 224/224 e 0 to 0 dl 1784949092 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949081.527887:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000077fe3ed0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289673216:12345-192.168.202.137@tcp:400: Request processed in 219us (854us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949081.527893:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 748 00000100:00000040:1.0:1784949081.527897:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949081.527900:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949081.527948:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949081.527953:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000004db9864. 00000020:00000010:1.0:1784949081.527958:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949081.527963:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000ee879bbd. 00000020:00000040:1.0:1784949081.527975:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949081.527979:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949081.528034:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949081.528041:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000bc928ba2. 00000400:00000200:0.0:1784949081.528048:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949081.528064:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949081.528071:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:0.0:1784949081.528074:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000800:00000001:1.0:1784949081.528077:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949081.528079:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949081.528083:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949081.528087:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949081.528091:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949081.528096:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949081.528102:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949081.528113:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ae00 00000400:00000200:1.0:1784949081.528121:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8d99 [1] + 224 00000800:00000001:1.0:1784949081.528125:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949081.528130:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949081.528132:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949081.528136:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949081.528143:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949081.528147:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949081.528156:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949081.528182:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949081.528189:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949081.528191:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.528228:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949081.528232:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949081.528236:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949081.528239:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949081.528241:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000010aeeaab. 00000400:00000200:0.0:1784949081.528246:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:0.0:1784949081.528248:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:0.0:1784949081.528251:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949081.528257:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949081.528269:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.528298:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949081.528300:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.528303:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949081.528305:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949081.528307:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949081.528310:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949081.528313:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949081.528320:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=98 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949081.528324:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=98 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949081.528328:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949081.528329:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949081.528331:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949081.528336:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949081.528345:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949081.528348:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949081.528350:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949081.528353:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949081.528354:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949081.528358:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.528365:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949081.528367:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949081.528369:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949081.528372:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949081.528380:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000001dee7260 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289673216:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949081.528386:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.528388:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949081.528390:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000001dee7260 x1871651289673216/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949081.528398:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949081.528400:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949081.528403:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:0.0:1784949081.528407:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949081.528408:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949081.528410:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949081.528412:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949081.528414:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000008d5370ec. 02000000:00000001:0.0:1784949081.528417:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949081.528419:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949081.528421:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000001dee7260. 00000100:00000001:0.0:1784949081.528424:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949081.528426:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.528428:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949081.528432:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949081.528434:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949081.528436:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949081.528438:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949081.569041:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306020908, at 4306020909 00000100:00000040:1.0:1784949081.569042:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306020908, at 4306020909 00000100:00000040:2.0:1784949081.569058:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306020908, at 4306020909 00000100:00000010:1.0:1784949081.569068:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:3.0:1784949081.569072:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:0.0:1784949081.569079:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306020908, at 4306020909 00000100:00000010:2.0:1784949081.569080:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000010:0.0:1784949081.569095:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000800:00000100:3.0:1784949081.716118:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000800:00000010:3.0:1784949081.716124:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a21d8ef3. 00000400:00000200:3.0:1784949081.716136:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949081.716144:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949081.716154:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11354 health :900 00000400:00000200:3.0:1784949081.716164:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949081.716172:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:3.0:1784949081.716179:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:1.0:1784949081.716245:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949081.716261:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:1.0:1784949081.716271:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949081.716294:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949081.716298:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949081.716301:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000400:00000200:1.0:1784949081.716309:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949081.716314:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:1.0:1784949081.716320:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a5bcc72e. 00000400:00000200:1.0:1784949081.716325:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:1.0:1784949081.716327:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:1.0:1784949081.716330:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00010000:00000001:2.0:1784949082.356097:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949082.356107:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949082.356115:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949082.356117:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949082.356252:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949082.356262:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949082.356267:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:3.0:1784949082.740076:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949082.740092:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000000e755589. 00000800:00000100:0.0:1784949082.740096:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:0.0:1784949082.740105:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000005678171. 00000400:00000010:3.0:1784949082.740107:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e7299d92. 00000400:00000200:0.0:1784949082.740115:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949082.740119:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000400:00000200:0.0:1784949082.740127:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949082.740136:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11355 health :800 00000400:00000200:0.0:1784949082.740143:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949082.740148:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000010:3.0:1784949082.740149:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000189c3332. 00000400:00000200:0.0:1784949082.740157:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:3.0:1784949082.740165:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949082.740202:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949082.740216:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949082.740224:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3584:3584:242:4294967295] 192.168.202.137@tcp LPNI seq info [2309:2309:8:4294967295] 00000400:00000200:3.0:1784949082.740235:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949082.740246:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949082.740256:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949082.740265:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000400:00000200:1.0:1784949082.740290:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784949082.740304:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949082.740325:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000800:00000200:3.0:1784949082.740335:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:1.0:1784949082.740337:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949082.740344:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:3.0:1784949082.740348:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949082.740350:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784949082.740355:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000400:00000200:1.0:1784949082.740369:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949082.740377:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:1.0:1784949082.740384:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009802f36f. 00000400:00000200:1.0:1784949082.740393:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:1.0:1784949082.740397:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:1.0:1784949082.740404:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000800:00000200:2.0:1784949082.740590:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949082.740591:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784949082.740598:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:2.0:1784949082.740606:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949082.740608:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949082.740616:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949082.740617:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949082.740627:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784949082.740630:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949082.740632:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:0.0:1784949082.740647:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949082.740664:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949082.740675:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949082.740686:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949082.740691:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3585:3585:243:4294967295] 192.168.202.137@tcp LPNI seq info [2310:2310:8:4294967295] 00000400:00000200:0.0:1784949082.740700:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949082.740706:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949082.740711:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000800:00000200:0.0:1784949082.740717:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949082.740723:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949082.740726:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949082.740797:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949082.740834:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949082.740842:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000400:00000200:0.0:1784949082.740852:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949082.740853:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949082.740859:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949082.740867:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949082.740870:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949082.740872:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949082.740881:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949082.740892:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949082.740956:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8dad 00000800:00000001:2.0:1784949082.740963:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949082.740969:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949082.740972:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949082.740975:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949082.740981:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:2.0:1784949082.740984:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000400:00000200:2.0:1784949082.740988:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949082.740991:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000000e755589. 00000400:00000010:2.0:1784949082.740994:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e7299d92. 00000800:00000001:2.0:1784949082.741002:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949082.741004:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949083.380064:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949083.380073:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949083.380082:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949083.380084:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949083.380159:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949083.380166:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949083.380171:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949084.404049:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949084.404060:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949084.404070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949084.404072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949084.404188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949084.404194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949084.404199:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1784949084.788012:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000800:00000010:0.0:1784949084.788018:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:0.0:1784949084.788028:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949084.788032:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949084.788039:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11357 health :900 00000400:00000200:0.0:1784949084.788044:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949084.788047:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:0.0:1784949084.788052:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:1.0:1784949084.788091:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949084.788098:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:1.0:1784949084.788104:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949084.788106:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949084.788109:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:1.0:1784949084.788112:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000400:00000200:1.0:1784949084.788115:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949084.788118:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:1.0:1784949084.788122:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000043d62367. 00000400:00000200:1.0:1784949084.788126:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:1.0:1784949084.788128:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:1.0:1784949084.788131:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00010000:00000001:2.0:1784949085.428077:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949085.428087:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949085.428095:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949085.428097:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949085.428295:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949085.428302:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949085.428306:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:3.0:1784949085.812080:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949085.812094:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000000d64870c. 00000400:00000010:3.0:1784949085.812108:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e7299d92. 00000400:00000010:3.0:1784949085.812117:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000467f058f. 00000400:00000010:3.0:1784949085.812128:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ae557b6e. 00000400:00000200:3.0:1784949085.812133:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949085.812150:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949085.812156:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949085.812161:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3586:3586:244:4294967295] 192.168.202.137@tcp LPNI seq info [2311:2311:8:4294967295] 00000400:00000200:3.0:1784949085.812169:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949085.812177:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949085.812183:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949085.812188:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003547cd5f. 00000800:00000200:3.0:1784949085.812196:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949085.812205:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949085.812210:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949085.812237:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11359 health :900 00000800:00000200:1.0:1784949085.812483:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949085.812494:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003547cd5f. 00000400:00000200:1.0:1784949085.812521:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949085.812547:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949085.812557:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784949085.812571:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784949085.812576:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949085.812587:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949085.812595:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784949085.812604:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949085.812628:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949085.812667:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949085.812685:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949085.812706:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949085.812716:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3587:3587:244:4294967295] 192.168.202.137@tcp LPNI seq info [2312:2312:8:4294967295] 00000400:00000200:0.0:1784949085.812736:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949085.812748:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949085.812757:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000800:00000200:0.0:1784949085.812769:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949085.812782:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949085.812789:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784949085.812880:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949085.813021:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949085.813029:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000400:00000200:0.0:1784949085.813037:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949085.813049:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949085.813057:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949085.813065:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949085.813076:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949085.813079:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949085.813083:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949085.813097:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949085.813116:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8db1 00000800:00000001:2.0:1784949085.813123:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949085.813134:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949085.813137:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949085.813142:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949085.813149:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000400:00000010:2.0:1784949085.813153:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000400:00000200:2.0:1784949085.813157:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949085.813161:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000000d64870c. 00000400:00000010:2.0:1784949085.813166:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e7299d92. 00000800:00000001:2.0:1784949085.813176:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949085.813178:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949086.452047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949086.452061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949086.452076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949086.452079:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949086.452186:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949086.452197:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949086.452204:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949086.644080:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:1.0:1784949086.644090:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949086.644103:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949086.644107:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949086.644114:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784949086.644120:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086a153f0. 00000100:00000001:2.0:1784949086.644124:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949086.644134:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1784949086.644136:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949086.644143:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949086.644146:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949086.644157:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949086.644157:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949086.644169:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949086.644185:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e4c24e27. 00000100:00000001:1.0:1784949086.644198:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949086.644215:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949086.644279:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949086.644284:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949086.644294:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a153f0] to pc [ptlrpcd_00_03+3] req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949086.644339:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949086.644342:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949086.644347:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949086.644351:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949086.644355:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949086.644358:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000047bf8f5f. 00000020:00000040:1.0:1784949086.644361:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949086.644365:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949086.644368:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949086.644371:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949086.644376:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b80ce7eb. 00000100:00000001:1.0:1784949086.644381:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.644387:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949086.644392:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1784949086.644396:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:2.0:1784949086.644405:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949086.644417:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949086.644418:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949086.644421:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.644423:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.644425:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949086.644429:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949086.644432:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047bf8f5f] to pc [ptlrpcd_00_00+0] req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949086.644436:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.644437:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949086.644444:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949086.644457:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949086.644472:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949086.644475:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949086.644480:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949086.644484:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949086.644485:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949086.644495:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949086.644498:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11363) 00000100:00100000:2.0:1784949086.644501:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000086a153f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289673344:0@lo:400: 00000100:00000001:0.0:1784949086.644504:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949086.644511:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1784949086.644513:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1784949086.644518:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949086.644522:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949086.644527:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1784949086.644528:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949086.644534:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.644539:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949086.644544:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949086.644553:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949086.644558:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:2.0:1784949086.644563:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007a6587e5. 00000100:00000040:0.0:1784949086.644569:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949086.644570:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949086.644588:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000919ac10c. 00000100:00000001:0.0:1784949086.644589:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949086.644594:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949086.644601:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:2.0:1784949086.644605:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 02000000:00000001:0.0:1784949086.644606:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949086.644608:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949086.644617:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949086.644618:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949086.644623:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289673344, portal 25 00000100:00100000:0.0:1784949086.644624:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000047bf8f5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289673472:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949086.644634:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949086.644635:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949086.644639:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637807360 : -115544071744256 : ffff96e9cca5d500) 02000000:00000001:0.0:1784949086.644642:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949086.644647:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949086.644659:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784949086.644663:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949086.644666:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949086.644673:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949086.644686:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 00000100:00000001:2.0:1784949086.644687:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949086.644697:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949086.644699:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000400:00000010:2.0:1784949086.644706:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000100:00000200:0.0:1784949086.644708:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289673472, portal 10 00000100:00000200:2.0:1784949086.644715:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289673344, offset 0 00000100:00000001:0.0:1784949086.644716:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949086.644720:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637805568 : -115544071746048 : ffff96e9cca5ce00) 00000400:00000200:2.0:1784949086.644732:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:0.0:1784949086.644736:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949086.644748:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949086.644752:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949086.644759:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000400:00000200:2.0:1784949086.644759:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:0.0:1784949086.644769:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289673472, offset 0 00000400:00000200:0.0:1784949086.644779:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949086.644785:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ae80 00000400:00000200:2.0:1784949086.644795:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8421 [2] + 0 00000400:00000200:0.0:1784949086.644799:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000010:2.0:1784949086.644807:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ba88f347. 00000400:00000200:0.0:1784949086.644812:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949086.644818:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000005c60caa0 00000400:00000200:0.0:1784949086.644820:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949086.644829:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949086.644832:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949086.644835:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949086.644841:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3588:3588:244:4294967295] 192.168.202.137@tcp LPNI seq info [2313:2313:8:4294967295] 00000400:00000200:2.0:1784949086.644850:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949086.644863:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949086.644871:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949086.644875:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005c60caa0 00000800:00000010:0.0:1784949086.644878:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f8a82730. 00000400:00000010:2.0:1784949086.644881:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000005c60caa0. 00000100:00000001:2.0:1784949086.644893:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:0.0:1784949086.644894:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:2.0:1784949086.644898:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949086.644955:0:337121:0:(events.c:337:request_in_callback()) incoming req@000000009e6f08df x1871651289673344 msgsize 224 00000100:00100000:2.0:1784949086.644965:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949086.644969:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:0.0:1784949086.644980:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784949086.644980:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000200:0.0:1784949086.644989:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f8a82730 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1784949086.644995:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:2.0:1784949086.645001:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000100:00000001:2.0:1784949086.645005:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949086.645012:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949086.645014:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949086.645023:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2161 usecs (3 jiffies) 00000100:00000001:0.0:1784949086.645034:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306025988 00000100:00000010:0.0:1784949086.645046:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:2.0:1784949086.645051:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949086.645060:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000040:2.0:1784949086.645064:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949086.645083:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306025988 00000100:00000001:1.0:1784949086.645093:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000010:0.0:1784949086.645094:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00100000:1.0:1784949086.645102:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289673344 00000100:00000001:2.0:1784949086.645106:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949086.645107:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 02000000:00000001:1.0:1784949086.645109:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949086.645118:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949086.645121:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949086.645122:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306025988 00000100:00000001:1.0:1784949086.645124:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949086.645135:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:2.0:1784949086.645138:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949086.645139:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949086.645142:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949086.645152:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306026027 00000100:00100000:1.0:1784949086.645153:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289673344 00000100:00000001:0.0:1784949086.645154:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306025988 00000020:00000001:1.0:1784949086.645164:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000010:0.0:1784949086.645168:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000020:00000040:1.0:1784949086.645170:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784949086.645173:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306025988 00000100:00000001:0.0:1784949086.645176:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000001:1.0:1784949086.645179:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949086.645189:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306026027 00000020:00000040:1.0:1784949086.645190:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949086.645191:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949086.645202:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949086.645204:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.645206:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306025988 00000020:00000001:1.0:1784949086.645214:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949086.645217:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.645220:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306026027 00000100:00000001:0.0:1784949086.645226:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949086.645227:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949086.645228:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306025988 00000100:00000001:2.0:1784949086.645289:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306026027 00000100:00000001:2.0:1784949086.645293:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306025988 00000100:00000001:2.0:1784949086.645301:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949086.645304:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.645308:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.645313:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949086.645347:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949086.645349:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949086.645357:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f8a82730. 00000400:00000200:3.0:1784949086.645366:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949086.645368:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949086.645378:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949086.645378:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949086.645385:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949086.645388:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:3.0:1784949086.645394:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000400:00000200:0.0:1784949086.645401:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949086.645404:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949086.645413:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949086.645421:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643af00 00000400:00000200:0.0:1784949086.645440:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 168816 00000100:00000001:3.0:1784949086.645441:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:0.0:1784949086.645451:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949086.645452:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949086.645463:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949086.645475:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949086.645478:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949086.645482:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1784949086.645483:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000a400f603. 00000400:00000200:0.0:1784949086.645493:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949086.645494:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949086.645501:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000a400f603. 00000100:00000001:0.0:1784949086.645503:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949086.645507:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949086.645515:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000073a82e6. 00000020:00000010:1.0:1784949086.645517:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ee879bbd. 00000100:00000040:0.0:1784949086.645520:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000073a82e6 x1871651289673472 msgsize 224 00000020:00000010:1.0:1784949086.645529:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000004db9864. 00000100:00100000:0.0:1784949086.645531:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000010:1.0:1784949086.645541:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949086.645554:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784949086.645557:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949086.645565:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949086.645567:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000001:0.0:1784949086.645568:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949086.645571:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949086.645576:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949086.645587:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949086.645599:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949086.645619:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949086.645622:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949086.645631:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 773 00000100:00000001:0.0:1784949086.645634:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:1.0:1784949086.645637:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949086.645642:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949086.645645:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782153808 : -115538927397808 : ffff96eaff465450) 00000100:00000001:0.0:1784949086.645647:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949086.645653:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949086.645664:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009e6f08df x1871651289673344/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:747/0 lens 224/0 e 0 to 0 dl 1784949097 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949086.645707:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949086.645712:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000a400f603. 00000100:00000001:1.0:1784949086.645716:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949086.645720:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000a400f603. 00000100:00100000:1.0:1784949086.645728:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009e6f08df pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289673344:12345-0@lo:400: 00000100:00000200:1.0:1784949086.645737:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289673344 00000020:00000001:1.0:1784949086.645745:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949086.645753:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949086.645758:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949086.645764:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949086.645768:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949086.645778:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949086.645778:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:1.0:1784949086.645788:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949086.645788:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289673472 02000000:00000001:3.0:1784949086.645797:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:1.0:1784949086.645801:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949086.645802:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:1.0:1784949086.645814:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949086.645814:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949086.645828:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949086.645829:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949086.645841:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:3.0:1784949086.645844:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289673472 00000100:00000001:1.0:1784949086.645847:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:3.0:1784949086.645849:0:342762:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784949086.645855:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000040:3.0:1784949086.645855:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949086.645865:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:1.0:1784949086.645870:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000095d8d756. 00000020:00000040:3.0:1784949086.645871:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:1.0:1784949086.645880:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949086.645881:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949086.645894:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949086.645895:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949086.645907:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949086.645911:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289673344 00010000:00000001:1.0:1784949086.645915:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949086.645923:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009e6f08df x1871651289673344/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:747/0 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949086.645940:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949086.645949:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000001:3.0:1784949086.645952:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949086.645955:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000020:00000010:3.0:1784949086.645964:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e7299d92. 00000020:00000010:3.0:1784949086.645969:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ff483bd2. 00000020:00000010:3.0:1784949086.645984:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00010000:00000001:1.0:1784949086.646000:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949086.646003:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949086.646004:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00001000:1.0:1784949086.646022:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=108 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949086.646023:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949086.646030:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949086.646043:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949086.646044:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949086.646053:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949086.646057:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949086.646058:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949086.646061:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 02000000:00000001:1.0:1784949086.646064:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784949086.646065:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949086.646071:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949086.646076:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949086.646080:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949086.646083:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949086.646085:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000100:00000200:1.0:1784949086.646095:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289673344, offset 224 00000100:00000001:3.0:1784949086.646100:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:1.0:1784949086.646103:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949086.646103:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949086.646113:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 749 00000400:00000200:1.0:1784949086.646126:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949086.646127:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949086.646139:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554968064 : -115544154583552 : ffff96e9c7b5ce00) 00000100:00000040:3.0:1784949086.646152:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000073a82e6 x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:747/0 lens 224/0 e 0 to 0 dl 1784949097 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949086.646153:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ae80 00000400:00000200:1.0:1784949086.646184:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8db5 [1] + 224 00000400:00000200:1.0:1784949086.646210:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949086.646212:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949086.646223:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000400:00000200:1.0:1784949086.646227:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949086.646246:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949086.646249:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000100:00100000:3.0:1784949086.646255:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000073a82e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289673472:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949086.646259:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289673472 00000020:00000001:3.0:1784949086.646263:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949086.646266:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784949086.646268:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949086.646271:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949086.646284:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949086.646287:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949086.646290:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000200:1.0:1784949086.646292:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949086.646296:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949086.646310:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949086.646316:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:1.0:1784949086.646324:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949086.646324:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949086.646339:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949086.646346:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949086.646353:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949086.646358:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000005c6a6527. 02000000:00000001:3.0:1784949086.646360:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949086.646364:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949086.646364:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949086.646369:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949086.646374:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289673472 00010000:00000001:3.0:1784949086.646377:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:1.0:1784949086.646380:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:1.0:1784949086.646389:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00010000:00000200:3.0:1784949086.646389:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000073a82e6 x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:747/0 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949086.646402:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949086.646407:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949086.646408:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949086.646421:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1784949086.646422:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949086.646424:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949086.646435:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949086.646437:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:2.0:1784949086.646440:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00001000:3.0:1784949086.646445:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=103 v=5 (1 1 1 1) 00000400:00000001:2.0:1784949086.646446:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:1.0:1784949086.646448:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:2.0:1784949086.646454:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000020:00000001:1.0:1784949086.646459:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949086.646461:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1784949086.646464:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000020:00000001:1.0:1784949086.646471:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949086.646472:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000400:00000010:2.0:1784949086.646473:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000100:00000040:1.0:1784949086.646486:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009e6f08df x1871651289673344/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:747/0 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949086.646489:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949086.646489:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949086.646503:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000200:2.0:1784949086.646509:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949086.646512:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949086.646529:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1784949086.646548:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009e6f08df pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289673344:12345-0@lo:400: Request processed in 828us (1578us total) trans 0 rc 0/0 00000100:00000040:3.0:1784949086.646550:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:2.0:1784949086.646552:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949086.646561:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000200:3.0:1784949086.646579:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289673472, offset 224 00000100:00100000:1.0:1784949086.646592:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 773 00000100:00000040:1.0:1784949086.646603:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949086.646606:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949086.646610:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949086.646615:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949086.646618:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949086.646619:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784949086.646632:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:1.0:1784949086.646635:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000004db9864. 02000000:00000001:2.0:1784949086.646636:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949086.646641:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:1.0:1784949086.646647:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:2.0:1784949086.646649:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949086.646654:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:2.0:1784949086.646663:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949086.646666:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000ee879bbd. 00000100:00001000:2.0:1784949086.646673:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=108 v=5 (5 5 5 5) 00000400:00000200:3.0:1784949086.646675:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3589:3589:244:4294967295] 192.168.202.137@tcp LPNI seq info [2314:2314:8:4294967295] 00000020:00000040:1.0:1784949086.646685:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:2.0:1784949086.646692:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=108 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949086.646704:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949086.646707:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949086.646709:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:2.0:1784949086.646717:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000010:1.0:1784949086.646720:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000095d8d756. 00010000:00000001:2.0:1784949086.646724:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949086.646735:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784949086.646736:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949086.646738:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949086.646753:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949086.646758:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949086.646769:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000200:1.0:1784949086.646773:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000800:00000010:3.0:1784949086.646785:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000400:00000010:1.0:1784949086.646789:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d40cdb47. 00000400:00000010:1.0:1784949086.646804:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bf496567. 00000100:00000001:2.0:1784949086.646805:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1784949086.646809:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949086.646818:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949086.646823:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949086.646832:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:2.0:1784949086.646835:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949086.646841:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00100000:2.0:1784949086.646865:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000086a153f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289673344:0@lo:400: 00000100:00000001:2.0:1784949086.646871:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949086.646876:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.646877:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949086.646886:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000086a153f0 x1871651289673344/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949086.646889:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949086.646901:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949086.646927:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949086.646930:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949086.646937:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000073a82e6 x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:747/0 lens 224/224 e 0 to 0 dl 1784949097 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949086.646966:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949086.646975:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949086.646983:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007a6587e5. 00000100:00100000:3.0:1784949086.646984:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000073a82e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289673472:12345-192.168.202.137@tcp:400: Request processed in 762us (1452us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949086.646998:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 749 02000000:00000001:2.0:1784949086.646999:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949086.647003:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000020:00000001:2.0:1784949086.647004:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949086.647008:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949086.647008:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949086.647013:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:2.0:1784949086.647019:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000010:3.0:1784949086.647021:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ff483bd2. 02000000:00000010:2.0:1784949086.647027:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e4c24e27. 00000020:00000010:3.0:1784949086.647028:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 02000000:00000001:2.0:1784949086.647048:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000010:3.0:1784949086.647048:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e7299d92. 02000000:00000001:2.0:1784949086.647057:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:3.0:1784949086.647061:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000010:2.0:1784949086.647063:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000086a153f0. 00000100:00000001:3.0:1784949086.647066:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949086.647078:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949086.647082:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949086.647104:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949086.647114:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949086.647118:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949086.647123:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:0.0:1784949086.647128:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949086.647128:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949086.647139:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000400:00000200:0.0:1784949086.647149:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949086.647156:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949086.647162:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949086.647170:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000800:00000001:1.0:1784949086.647172:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949086.647174:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000800:00000001:1.0:1784949086.647180:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949086.647184:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784949086.647187:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.647190:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949086.647204:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949086.647219:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643af00 00000400:00000200:1.0:1784949086.647227:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8db9 [1] + 224 00000800:00000001:1.0:1784949086.647277:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949086.647289:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949086.647293:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949086.647297:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949086.647302:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949086.647307:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949086.647318:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949086.647338:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949086.647345:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949086.647348:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.647386:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949086.647391:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949086.647397:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949086.647399:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949086.647402:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000400:00000200:0.0:1784949086.647407:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:0.0:1784949086.647409:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000100:00000001:0.0:1784949086.647412:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949086.647418:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949086.647430:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949086.647440:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949086.647442:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.647444:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949086.647446:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949086.647448:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949086.647450:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949086.647452:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949086.647458:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=103 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949086.647462:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=103 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949086.647465:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949086.647466:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949086.647469:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949086.647472:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949086.647482:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949086.647486:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949086.647488:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949086.647491:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949086.647492:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949086.647495:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949086.647502:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949086.647505:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949086.647506:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949086.647509:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949086.647517:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000047bf8f5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289673472:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949086.647521:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.647523:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949086.647526:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000047bf8f5f x1871651289673472/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949086.647534:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949086.647536:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949086.647538:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dc7a25c6. 02000000:00000001:0.0:1784949086.647551:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949086.647553:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949086.647554:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949086.647556:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949086.647558:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b80ce7eb. 02000000:00000001:0.0:1784949086.647561:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949086.647563:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949086.647564:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000047bf8f5f. 00000100:00000001:0.0:1784949086.647568:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949086.647569:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.647571:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949086.647577:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949086.647579:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949086.647581:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949086.647583:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949086.687977:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306026027, at 4306026028 00000100:00000010:3.0:1784949086.687989:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000040:1.0:1784949086.688002:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306026027, at 4306026028 00000100:00000010:1.0:1784949086.688023:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000040:0.0:1784949086.688057:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306026027, at 4306026028 00000100:00000040:2.0:1784949086.688065:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306026027, at 4306026028 00000100:00000010:0.0:1784949086.688070:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000010:2.0:1784949086.688084:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000800:00000100:0.0:1784949086.836064:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.5@tcp 00000800:00000010:0.0:1784949086.836071:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:0.0:1784949086.836084:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949086.836089:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949086.836097:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11359 health :900 00000400:00000200:0.0:1784949086.836102:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949086.836107:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:0.0:1784949086.836111:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000800:00000100:0.0:1784949086.836138:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.10@tcp 00000800:00000010:0.0:1784949086.836140:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007a3f8609. 00000400:00000200:0.0:1784949086.836143:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949086.836147:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949086.836150:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949086.836153:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:0.0:1784949086.836156:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:1.0:1784949086.836177:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949086.836188:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 00000400:00000001:1.0:1784949086.836196:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949086.836199:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949086.836203:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:1.0:1784949086.836207:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000400:00000200:1.0:1784949086.836212:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949086.836216:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:1.0:1784949086.836221:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000072ce5ef0. 00000400:00000200:1.0:1784949086.836228:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-110 00000400:00000200:1.0:1784949086.836231:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -110 00000400:00000200:1.0:1784949086.836257:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:1.0:1784949086.836261:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:1.0:1784949086.836263:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949086.836264:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949086.836265:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:1.0:1784949086.836267:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:1.0:1784949086.836269:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949086.836271:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:1.0:1784949086.836284:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000bbdbdf4. 00000400:00000200:1.0:1784949086.836286:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-110 00000400:00000200:1.0:1784949086.836289:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -110 00000400:00000200:1.0:1784949086.836291:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00010000:00000001:2.0:1784949087.476088:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949087.476098:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949087.476107:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949087.476109:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949087.476304:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949087.476365:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949087.476370:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949087.667979:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949087.667991:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949087.667993:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949087.667997:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949087.667999:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949087.668002:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949087.668003:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949087.860052:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949087.860067:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000075d9a982. 00000400:00000010:3.0:1784949087.860079:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e7299d92. 00000400:00000010:3.0:1784949087.860089:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 00000800:00000100:0.0:1784949087.860094:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.9@tcp 00000400:00000010:3.0:1784949087.860101:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000325e9e7. 00000800:00000010:0.0:1784949087.860106:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cf155122. 00000400:00000200:3.0:1784949087.860121:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949087.860130:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949087.860142:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949087.860203:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949087.860212:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949087.860217:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3590:3590:246:4294967295] 192.168.202.137@tcp LPNI seq info [2315:2315:8:4294967295] 00000400:00000200:3.0:1784949087.860278:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949087.860284:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949087.860289:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949087.860293:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000400:00000200:0.0:1784949087.860296:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11360 health :700 00000800:00000200:3.0:1784949087.860304:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949087.860305:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949087.860319:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000800:00000200:3.0:1784949087.860330:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:0.0:1784949087.860336:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000800:00000200:3.0:1784949087.860337:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 2, nob 96 niov 1 nkiov 0 00000800:00000100:0.0:1784949087.860389:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000800:00000010:0.0:1784949087.860393:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000400:00000200:0.0:1784949087.860402:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949087.860403:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949087.860421:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000200:0.0:1784949087.860427:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000001:1.0:1784949087.860430:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949087.860435:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949087.860439:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0c9b918 00000400:00000010:1.0:1784949087.860441:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f0c9b918. 00000400:00000200:1.0:1784949087.860446:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949087.860447:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949087.860454:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:1.0:1784949087.860459:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032fe57fd. 00000400:00000200:0.0:1784949087.860460:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:1.0:1784949087.860469:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:0.0:1784949087.860473:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:1.0:1784949087.860480:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000400:00000200:1.0:1784949087.860487:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000400:00000200:1.0:1784949087.860499:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:1.0:1784949087.860506:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949087.860509:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000100:0.0:1784949087.860510:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000400:00000200:1.0:1784949087.860512:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000800:00000001:3.0:1784949087.860514:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784949087.860517:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000400:00000200:1.0:1784949087.860523:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000200:2.0:1784949087.860523:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949087.860527:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949087.860529:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000800:00000010:0.0:1784949087.860533:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c2b2d4f6. 00000400:00000010:1.0:1784949087.860534:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b1cbf66f. 00000800:00000010:2.0:1784949087.860534:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000800:00000001:3.0:1784949087.860534:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949087.860542:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949087.860554:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949087.860556:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949087.860556:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949087.860557:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:1.0:1784949087.860572:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000400:00000200:3.0:1784949087.860574:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784949087.860587:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00000400:00000200:0.0:1784949087.860593:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949087.860594:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949087.860594:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949087.860630:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949087.860633:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949087.860643:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949087.860649:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:0.0:1784949087.860650:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:3.0:1784949087.860665:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3591:3591:247:4294967295] 192.168.202.137@tcp LPNI seq info [2316:2316:7:4294967295] 00000400:00000200:3.0:1784949087.860676:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949087.860682:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949087.860688:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000800:00000010:3.0:1784949087.860690:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784949087.860701:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949087.860708:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949087.860712:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949087.860740:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949087.860799:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949087.860807:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:0.0:1784949087.860814:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949087.860816:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949087.860819:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b039801a 00000400:00000010:0.0:1784949087.860822:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b039801a. 00000400:00000200:0.0:1784949087.860825:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949087.860828:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:0.0:1784949087.860832:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009f047fe3. 00000400:00000200:0.0:1784949087.860839:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:0.0:1784949087.860843:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:0.0:1784949087.860848:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000800:00000001:1.0:1784949087.860863:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949087.860866:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949087.860871:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000800:00000001:1.0:1784949087.860872:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949087.860876:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949087.860879:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949087.860883:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949087.860890:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949087.860894:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000001:3.0:1784949087.860895:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949087.860936:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8dd1 00000800:00000001:1.0:1784949087.860941:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949087.860947:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949087.860951:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949087.860956:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949087.860963:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000400:00000010:1.0:1784949087.860964:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 00000400:00000200:1.0:1784949087.860966:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949087.860968:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000075d9a982. 00000400:00000010:1.0:1784949087.860970:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e7299d92. 00000800:00000001:1.0:1784949087.860976:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949087.860978:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949088.499998:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949088.500007:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949088.500014:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949088.500016:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949088.500142:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949088.500147:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949088.500151:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949088.691988:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949088.692004:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949088.692007:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949088.692013:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949088.692016:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949088.692020:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949088.692022:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949088.884135:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000800:00000010:1.0:1784949088.884143:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cee99234. 00000400:00000200:1.0:1784949088.884151:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949088.884159:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949088.884178:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11361 health :900 00000400:00000200:1.0:1784949088.884185:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949088.884190:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:1.0:1784949088.884195:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:0.0:1784949088.884303:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949088.884320:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:0.0:1784949088.884330:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949088.884332:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949088.884336:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000400:00000010:0.0:1784949088.884338:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000400:00000200:0.0:1784949088.884346:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949088.884351:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:0.0:1784949088.884357:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000072ce2c8. 00000400:00000200:0.0:1784949088.884363:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:0.0:1784949088.884365:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:0.0:1784949088.884369:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00010000:00000001:2.0:1784949089.524038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949089.524059:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949089.524068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949089.524070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949089.524275:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949089.524285:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949089.524290:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949089.715997:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949089.716013:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949089.716015:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949089.716022:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949089.716026:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949089.716030:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949089.716033:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949089.908004:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949089.908016:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000f71d393a. 00000400:00000010:3.0:1784949089.908029:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e7299d92. 00000400:00000010:3.0:1784949089.908034:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000400:00000010:3.0:1784949089.908043:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ca3adacb. 00000400:00000200:3.0:1784949089.908055:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949089.908069:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949089.908075:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949089.908078:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3592:3592:250:4294967295] 192.168.202.137@tcp LPNI seq info [2317:2317:8:4294967295] 00000400:00000200:3.0:1784949089.908084:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949089.908090:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949089.908095:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949089.908100:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000800:00000200:3.0:1784949089.908106:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949089.908114:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949089.908118:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949089.908140:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11363 health :900 00000800:00000200:0.0:1784949089.908338:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949089.908348:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:0.0:1784949089.908361:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949089.908375:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784949089.908380:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949089.908384:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949089.908389:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949089.908398:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949089.908403:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949089.908410:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949089.908423:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949089.908436:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949089.908445:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949089.908454:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949089.908458:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3593:3593:250:4294967295] 192.168.202.137@tcp LPNI seq info [2318:2318:8:4294967295] 00000400:00000200:2.0:1784949089.908466:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949089.908470:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949089.908474:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fd30a9ad. 00000800:00000200:2.0:1784949089.908479:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949089.908483:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949089.908486:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fd30a9ad type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949089.908556:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949089.908628:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949089.908635:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fd30a9ad. 00000400:00000200:2.0:1784949089.908643:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949089.908645:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949089.908655:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949089.908662:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949089.908667:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949089.908669:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949089.908673:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949089.908685:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949089.908706:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8dd5 00000800:00000001:0.0:1784949089.908714:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949089.908724:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949089.908726:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949089.908730:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949089.908736:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:0.0:1784949089.908741:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000400:00000200:0.0:1784949089.908746:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949089.908765:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000f71d393a. 00000400:00000010:0.0:1784949089.908772:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e7299d92. 00000800:00000001:0.0:1784949089.908787:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949089.908792:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949090.548051:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949090.548131:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949090.548140:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949090.548142:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949090.548255:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949090.548260:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949090.548264:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949090.740095:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949090.740119:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949090.740122:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949090.740131:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949090.740137:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949090.740142:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949090.741842:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949090.932084:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.28@tcp 00000800:00000010:1.0:1784949090.932093:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004b264730. 00000400:00000200:1.0:1784949090.932106:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949090.932118:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.28@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949090.932128:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11363 health :900 00000400:00000200:1.0:1784949090.932135:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949090.932139:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.28@tcp: 1 00000400:00000200:1.0:1784949090.932145:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.28@tcp: -114 00000800:00000100:1.0:1784949090.932182:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.4@tcp 00000800:00000010:1.0:1784949090.932185:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:1.0:1784949090.932193:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949090.932228:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949090.932237:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949090.932240:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949090.932256:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:0.0:1784949090.932262:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.28@tcp(0000000015713dee) state 0x840 00000400:00000200:1.0:1784949090.932277:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000001:0.0:1784949090.932283:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.932288:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.932293:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:0.0:1784949090.932298:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000400:00000200:0.0:1784949090.932307:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.932327:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.28@tcp: -114 00000400:00000010:0.0:1784949090.932335:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004f865a97. 00000400:00000200:0.0:1784949090.932344:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.28@tcp:-110 00000400:00000200:0.0:1784949090.932352:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.28@tcp(0000000015713dee) state 0x2040 rc -110 00000400:00000200:0.0:1784949090.932357:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.28@tcp 00000400:00000200:0.0:1784949090.932377:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:0.0:1784949090.932381:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.932383:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.932386:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:0.0:1784949090.932388:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000400:00000200:0.0:1784949090.932390:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.932393:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:0.0:1784949090.932396:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf393d23. 00000400:00000200:0.0:1784949090.932401:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-110 00000400:00000200:0.0:1784949090.932404:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -110 00000400:00000200:0.0:1784949090.932408:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:3.0:1784949090.932447:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.28@tcp NID 111.222.173.28@tcp: -110. discovery complete 00000400:00000200:3.0:1784949090.932459:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.173.28@tcp primary NID 111.222.173.28@tcp rc -110 00000100:00000010:3.0:1784949090.932492:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000226948e6. 00000100:00000001:3.0:1784949090.932531:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784949090.932535:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000226948e6 refcount 1 to 111.222.173.28@tcp 00010000:00000010:3.0:1784949090.932542:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000077930f7d. 00010000:00080000:3.0:1784949090.932607:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.173.28@tcp at tail 00010000:00000001:3.0:1784949090.932611:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949090.932617:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949090.932674:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 0000000058a72386. 00000100:00000400:3.0:1784949090.932682:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.142.28@tcp: rc = -75 00000100:00000040:3.0:1784949090.932686:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.142.28@tcp cnt=32 00000400:00000200:3.0:1784949090.932690:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784949090.932694:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784949090.932699:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784949090.932702:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784949090.932704:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784949090.932714:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784949090.932722:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932725:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932729:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784949090.932731:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784949090.932737:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.932757:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784949090.932760:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.932762:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932765:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932768:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784949090.932770:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784949090.932774:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.932778:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784949090.932780:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.932782:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932784:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932787:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784949090.932789:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784949090.932793:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.932798:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784949090.932800:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.932801:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932804:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932807:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784949090.932809:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:3.0:1784949090.932813:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.932816:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:3.0:1784949090.932818:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.932820:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932822:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932824:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784949090.932826:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784949090.932829:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.932834:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:3.0:1784949090.932836:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.932838:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932842:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932845:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:3.0:1784949090.932847:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000200:3.0:1784949090.932853:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.932860:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000400:00000200:3.0:1784949090.932862:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.932867:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932871:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932874:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:3.0:1784949090.932875:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:3.0:1784949090.932879:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.932883:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949090.932889:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784949090.932893:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.932898:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1784949090.932933:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784949090.932953:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932957:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784949090.932960:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:3.0:1784949090.932963:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000010:0.0:1784949090.932966:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dc7a25c6. 00000400:00000200:3.0:1784949090.932974:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:3.0:1784949090.932978:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:0.0:1784949090.932980:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000400:00000200:3.0:1784949090.932982:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.932991:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.932995:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784949090.932997:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000010:0.0:1784949090.932999:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007864ed56. 00000400:00000200:3.0:1784949090.933003:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933018:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949090.933019:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:3.0:1784949090.933029:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933039:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933043:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933046:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784949090.933048:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:3.0:1784949090.933055:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933061:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:0.0:1784949090.933070:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.933076:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.933082:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.933086:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:0.0:1784949090.933093:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3594:3594:252:4294967295] 111.222.173.1@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:0.0:1784949090.933103:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:0.0:1784949090.933113:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.933123:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949090.933128:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000200:3.0:1784949090.933133:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:0.0:1784949090.933134:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000000d1fbdb0. 00000400:00000200:3.0:1784949090.933136:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:0.0:1784949090.933153:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949090.933158:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933164:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000800:00000200:0.1:1784949090.933168:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949090.933170:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000200:3.0:1784949090.933177:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.933181:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:3.0:1784949090.933183:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933185:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933188:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933190:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784949090.933191:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:3.0:1784949090.933195:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.933220:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784949090.933229:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933231:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1784949090.933238:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:3.0:1784949090.933240:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933246:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:0.0:1784949090.933249:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:3.0:1784949090.933254:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:3.0:1784949090.933265:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933268:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784949090.933280:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:3.0:1784949090.933283:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933289:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:0.0:1784949090.933292:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001879354a. 00000400:00000200:3.0:1784949090.933293:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000010:0.0:1784949090.933304:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000400:00000200:3.0:1784949090.933305:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:3.0:1784949090.933313:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000010:0.0:1784949090.933318:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001542e92a. 00000400:00000200:3.0:1784949090.933320:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933331:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949090.933335:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784949090.933339:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933344:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933347:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933350:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:3.0:1784949090.933353:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:0.0:1784949090.933358:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784949090.933363:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933371:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.933376:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.933379:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.933382:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:0.0:1784949090.933385:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3595:3595:251:4294967295] 111.222.142.1@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:0.0:1784949090.933390:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:0.0:1784949090.933396:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.933403:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000800:00000010:0.0:1784949090.933407:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f64a2f6c. 00000400:00000200:3.0:1784949090.933410:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784949090.933416:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933421:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:0.0:1784949090.933422:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949090.933432:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933439:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000800:00000200:0.1:1784949090.933440:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949090.933448:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:3.0:1784949090.933455:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.933459:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:3.0:1784949090.933461:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933464:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933467:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933469:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:3.0:1784949090.933471:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:3.0:1784949090.933475:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933485:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:3.0:1784949090.933486:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784949090.933492:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1784949090.933497:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:3.0:1784949090.933499:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933509:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933514:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784949090.933518:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:0.0:1784949090.933522:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784949090.933529:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000010:0.0:1784949090.933533:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000053ab16eb. 00000400:00000010:0.0:1784949090.933543:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f43fd4d9. 00000400:00000200:3.0:1784949090.933557:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:3.0:1784949090.933561:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:0.0:1784949090.933569:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007f814b08. 00000400:00000200:3.0:1784949090.933570:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933581:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933586:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:0.0:1784949090.933593:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784949090.933596:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:3.0:1784949090.933607:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933613:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:0.0:1784949090.933620:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.933624:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.933628:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.933632:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:0.0:1784949090.933636:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3596:3596:250:4294967295] 111.222.173.2@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:0.0:1784949090.933641:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:0.0:1784949090.933651:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.933658:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:0.0:1784949090.933663:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000595fa8f8. 00000400:00000200:3.0:1784949090.933664:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:3.0:1784949090.933672:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933676:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:0.0:1784949090.933677:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949090.933687:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000800:00000200:0.0:1784949090.933692:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784949090.933696:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784949090.933704:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000800:00000200:0.1:1784949090.933710:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:0.1:1784949090.933717:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784949090.933719:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000800:00000100:0.0:1784949090.933737:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784949090.933740:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000800:00000010:0.0:1784949090.933743:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000595fa8f8. 00000400:00000200:3.0:1784949090.933747:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1784949090.933752:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949090.933753:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1784949090.933769:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949090.933771:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933779:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:3.0:1784949090.933782:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:0.0:1784949090.933783:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949090.933789:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933795:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:3.0:1784949090.933806:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:3.0:1784949090.933809:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1784949090.933818:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:3.0:1784949090.933825:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933831:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933835:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:3.0:1784949090.933839:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:0.0:1784949090.933846:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:3.0:1784949090.933850:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.933857:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 rc 0 00000400:00000200:3.0:1784949090.933864:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784949090.933868:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.933878:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1784949090.933884:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:3.0:1784949090.933892:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.933898:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:3.0:1784949090.933901:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000010:0.0:1784949090.933903:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fff5e130. 00000400:00000200:3.0:1784949090.933946:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000010:0.0:1784949090.933959:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 00000400:00000200:3.0:1784949090.933960:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784949090.933969:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:0.0:1784949090.933976:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002026f8a1. 00000400:00000200:3.0:1784949090.933977:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.933986:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:0.0:1784949090.933994:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949090.933998:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:3.0:1784949090.934005:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:0.0:1784949090.934017:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:3.0:1784949090.934018:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.934031:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.934040:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.934043:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.934047:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:0.0:1784949090.934051:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3597:3597:250:4294967295] 111.222.142.2@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:0.0:1784949090.934059:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:0.0:1784949090.934071:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.934079:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000800:00000010:0.0:1784949090.934083:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000595fa8f8. 00000400:00000200:3.0:1784949090.934088:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:3.0:1784949090.934095:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:0.0:1784949090.934102:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949090.934104:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:0.1:1784949090.934112:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949090.934116:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.934123:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:0.0:1784949090.934126:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:3.0:1784949090.934132:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:0.0:1784949090.934138:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x261 rc 0 00000400:00000200:3.0:1784949090.934147:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.934149:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000010:0.0:1784949090.934163:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c73b3f00. 00000400:00000010:0.0:1784949090.934169:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000400:00000200:3.0:1784949090.934180:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:3.0:1784949090.934184:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:0.0:1784949090.934189:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000771e09fe. 00000400:00000200:3.0:1784949090.934190:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.934216:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.934221:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000200:3.0:1784949090.934224:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000200:0.0:1784949090.934232:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784949090.934234:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.934251:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:0.0:1784949090.934256:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.934259:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.934262:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.934266:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:0.0:1784949090.934269:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3598:3598:249:4294967295] 111.222.142.3@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:0.0:1784949090.934274:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:0.0:1784949090.934280:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.934286:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:0.0:1784949090.934291:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008746b1e6. 00000400:00000200:3.0:1784949090.934292:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784949090.934300:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.934305:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:0.0:1784949090.934306:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949090.934317:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000800:00000200:0.1:1784949090.934326:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949090.934328:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:3.0:1784949090.934332:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:3.0:1784949090.934336:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.934340:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:3.0:1784949090.934343:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.934346:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.934360:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.934364:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000200:3.0:1784949090.934367:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:3.0:1784949090.934371:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:0.0:1784949090.934383:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:3.0:1784949090.934389:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:0.0:1784949090.934395:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x261 rc 0 00000400:00000200:3.0:1784949090.934397:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.934405:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.934413:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.934416:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:3.0:1784949090.934421:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:0.0:1784949090.934424:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784949090.934432:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000010:0.0:1784949090.934435:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001df3b40e. 00000400:00000010:0.0:1784949090.934444:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000400:00000200:3.0:1784949090.934458:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784949090.934463:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:0.0:1784949090.934468:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001bae6561. 00000400:00000200:3.0:1784949090.934469:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.934485:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:0.0:1784949090.934487:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784949090.934494:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784949090.934498:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000200:3.0:1784949090.934503:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949090.934636:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000200:3.0:1784949090.934640:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949090.934647:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949090.934652:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949090.934657:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000100:00000010:3.0:1784949090.934664:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 0000000058a72386. 00000400:00000200:0.0:1784949090.934667:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:0.0:1784949090.934679:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784949090.934683:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000200:0.0:1784949090.934689:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00010000:00000001:3.0:1784949090.934694:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000400:00000200:0.0:1784949090.934695:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784949090.934703:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:0.0:1784949090.934707:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:0.0:1784949090.934715:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3599:3599:248:4294967295] 111.222.173.4@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:0.0:1784949090.934722:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:0.0:1784949090.934742:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.934753:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000400:00000010:3.0:1784949090.934757:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000e7299d92. 00000800:00000010:0.0:1784949090.934759:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a3cf4688. 00000400:00000200:3.0:1784949090.934770:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000e7299d92 nid 111.222.142.28@tcp 00000800:00000200:0.0:1784949090.934779:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000010:3.0:1784949090.934782:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000006204bf6f. 00000800:00000200:0.0:1784949090.934784:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784949090.934791:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 000000006204bf6f net tcp 00000400:00000010:3.0:1784949090.934801:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 0000000061851253. 00000800:00000200:0.1:1784949090.934801:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:0.1:1784949090.934812:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1784949090.934817:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 0000000061851253 nid 111.222.142.28@tcp 00000800:00000100:0.0:1784949090.934825:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.4@tcp 00000800:00000010:0.0:1784949090.934828:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a3cf4688. 00000400:00000200:0.0:1784949090.934833:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949090.934837:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.142.28@tcp NID 111.222.142.28@tcp flags 0x0 00000400:00000200:0.0:1784949090.934849:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949090.934853:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.142.28@tcp: 0 00000400:00000200:0.0:1784949090.934863:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949090.934873:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:3.0:1784949090.934879:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.28@tcp: 0 00000400:00000200:3.0:1784949090.934881:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1784949090.934889:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:0.0:1784949090.934974:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:0.0:1784949090.934979:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 rc 0 00000400:00000200:0.0:1784949090.934988:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000010:0.0:1784949090.934995:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003cdea007. 00000400:00000010:0.0:1784949090.935002:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000400:00000010:0.0:1784949090.935014:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a0e11faa. 00000400:00000200:0.0:1784949090.935025:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:0.0:1784949090.935039:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:0.0:1784949090.935047:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935052:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935058:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935064:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:0.0:1784949090.935074:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3600:3600:248:4294967295] 111.222.142.4@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:0.0:1784949090.935082:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:0.0:1784949090.935098:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935105:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:0.0:1784949090.935109:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a3cf4688. 00000800:00000200:0.0:1784949090.935114:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:0.1:1784949090.935119:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:0.0:1784949090.935124:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:0.0:1784949090.935126:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x261 rc 0 00000400:00000200:0.0:1784949090.935130:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000010:0.0:1784949090.935137:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001c9428b0. 00000400:00000010:0.0:1784949090.935139:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000236111b7. 00000400:00000010:0.0:1784949090.935143:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005c4dcc18. 00000400:00000200:0.0:1784949090.935146:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:0.0:1784949090.935150:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:0.0:1784949090.935154:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935157:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935159:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935162:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:0.0:1784949090.935166:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3601:3601:247:4294967295] 111.222.173.5@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:0.0:1784949090.935170:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:0.0:1784949090.935176:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935181:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000800:00000010:0.0:1784949090.935184:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b84187a1. 00000800:00000200:0.0:1784949090.935188:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:0.1:1784949090.935191:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:0.0:1784949090.935194:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:0.0:1784949090.935196:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x261 rc 0 00000400:00000200:0.0:1784949090.935222:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:0.0:1784949090.935227:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000307ddfde. 00000400:00000010:0.0:1784949090.935230:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000400:00000010:0.0:1784949090.935234:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fee11282. 00000400:00000200:0.0:1784949090.935238:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:0.0:1784949090.935244:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:0.0:1784949090.935248:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935252:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935256:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935259:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:0.0:1784949090.935262:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3602:3602:246:4294967295] 111.222.142.5@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:0.0:1784949090.935267:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:0.0:1784949090.935274:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935278:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:0.0:1784949090.935281:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000086e4d21. 00000800:00000200:0.0:1784949090.935286:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:0.1:1784949090.935289:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:0.0:1784949090.935306:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:0.0:1784949090.935309:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:0.0:1784949090.935312:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:0.0:1784949090.935322:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002fbb6924. 00000400:00000010:0.0:1784949090.935325:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000400:00000010:0.0:1784949090.935329:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004d156103. 00000400:00000200:0.0:1784949090.935332:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:0.0:1784949090.935337:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:0.0:1784949090.935342:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935345:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935348:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935352:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:0.0:1784949090.935360:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3603:3603:245:4294967295] 111.222.173.6@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:0.0:1784949090.935366:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:0.0:1784949090.935386:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935392:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:0.0:1784949090.935398:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d1b12746. 00000800:00000200:0.0:1784949090.935402:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.0:1784949090.935406:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784949090.935413:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:0.1:1784949090.935416:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1784949090.935424:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.6@tcp 00000800:00000010:0.0:1784949090.935427:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d1b12746. 00000400:00000200:0.0:1784949090.935432:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784949090.935437:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784949090.935441:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949090.935449:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:0.0:1784949090.935454:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:0.0:1784949090.935466:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:0.0:1784949090.935469:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 rc 0 00000400:00000200:0.0:1784949090.935473:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:0.0:1784949090.935477:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f11303f8. 00000400:00000010:0.0:1784949090.935481:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000400:00000010:0.0:1784949090.935490:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bfff640a. 00000400:00000200:0.0:1784949090.935495:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:0.0:1784949090.935499:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:0.0:1784949090.935503:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935507:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935510:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935513:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:0.0:1784949090.935517:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3604:3604:245:4294967295] 111.222.142.6@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:0.0:1784949090.935521:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:0.0:1784949090.935529:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935533:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:0.0:1784949090.935536:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d1b12746. 00000800:00000200:0.0:1784949090.935541:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:0.1:1784949090.935544:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000200:0.0:1784949090.935547:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:0.0:1784949090.935549:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x261 rc 0 00000400:00000200:0.0:1784949090.935553:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:0.0:1784949090.935560:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002c1a3d41. 00000400:00000010:0.0:1784949090.935570:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 00000400:00000010:0.0:1784949090.935576:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000098ebc0ae. 00000400:00000200:0.0:1784949090.935582:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:0.0:1784949090.935594:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:0.0:1784949090.935600:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935604:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935611:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935616:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:0.0:1784949090.935620:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3605:3605:244:4294967295] 111.222.142.7@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:0.0:1784949090.935626:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:0.0:1784949090.935631:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935636:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:0.0:1784949090.935641:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000006487a147. 00000800:00000200:0.0:1784949090.935648:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:0.1:1784949090.935657:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:0.0:1784949090.935662:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:0.0:1784949090.935664:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:0.0:1784949090.935667:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:0.0:1784949090.935670:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000047e19e91. 00000400:00000010:0.0:1784949090.935674:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000400:00000010:0.0:1784949090.935677:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000b1d42ed. 00000400:00000200:0.0:1784949090.935680:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:0.0:1784949090.935684:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:0.0:1784949090.935687:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935689:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935691:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935694:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:0.0:1784949090.935697:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3606:3606:243:4294967295] 111.222.173.8@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:0.0:1784949090.935701:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:0.0:1784949090.935706:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935711:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:0.0:1784949090.935714:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002aeee8e9. 00000800:00000200:0.0:1784949090.935719:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.0:1784949090.935722:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784949090.935730:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:0.1:1784949090.935733:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1784949090.935738:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.8@tcp 00000800:00000010:0.0:1784949090.935740:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002aeee8e9. 00000400:00000200:0.0:1784949090.935744:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784949090.935747:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784949090.935751:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949090.935754:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:0.0:1784949090.935757:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:0.0:1784949090.935763:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:0.0:1784949090.935765:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 rc 0 00000400:00000200:0.0:1784949090.935769:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:0.0:1784949090.935775:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000013f68999. 00000400:00000010:0.0:1784949090.935780:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000089c8262f. 00000400:00000010:0.0:1784949090.935786:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e69b1edb. 00000400:00000200:0.0:1784949090.935792:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:0.0:1784949090.935798:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:0.0:1784949090.935808:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935811:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935815:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935817:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:0.0:1784949090.935826:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3607:3607:243:4294967295] 111.222.173.9@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:0.0:1784949090.935831:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:0.0:1784949090.935838:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935853:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:0.0:1784949090.935858:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002aeee8e9. 00000800:00000200:0.0:1784949090.935864:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:0.1:1784949090.935868:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000400:00000200:0.0:1784949090.935872:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:0.0:1784949090.935874:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x261 rc 0 00000400:00000200:0.0:1784949090.935877:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:0.0:1784949090.935882:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000461737cb. 00000400:00000010:0.0:1784949090.935886:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00000400:00000010:0.0:1784949090.935889:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d5bb2c49. 00000400:00000200:0.0:1784949090.935939:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:0.0:1784949090.935944:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:0.0:1784949090.935947:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.935950:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.935953:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.935956:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:0.0:1784949090.935959:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3608:3608:242:4294967295] 111.222.142.9@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:0.0:1784949090.935964:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:0.0:1784949090.935971:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.935975:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:0.0:1784949090.935979:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000eafb2cdc. 00000800:00000200:0.0:1784949090.935984:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:0.1:1784949090.935988:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:0.0:1784949090.936004:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:0.0:1784949090.936007:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:0.0:1784949090.936011:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:0.0:1784949090.936016:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef8e57c8. 00000400:00000010:0.0:1784949090.936022:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000400:00000010:0.0:1784949090.936027:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a2b37178. 00000400:00000200:0.0:1784949090.936032:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:0.0:1784949090.936039:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:0.0:1784949090.936045:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.936051:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.936058:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.936062:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:0.0:1784949090.936066:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3609:3609:241:4294967295] 111.222.173.10@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:0.0:1784949090.936072:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:0.0:1784949090.936079:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.936085:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:0.0:1784949090.936090:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b51ad697. 00000800:00000200:0.0:1784949090.936095:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.0:1784949090.936099:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784949090.936106:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:0.1:1784949090.936110:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1784949090.936115:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.10@tcp 00000800:00000010:0.0:1784949090.936117:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b51ad697. 00000400:00000200:0.0:1784949090.936122:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784949090.936129:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784949090.936132:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949090.936135:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:0.0:1784949090.936138:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:0.0:1784949090.936145:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:0.0:1784949090.936148:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 rc 0 00000400:00000200:0.0:1784949090.936156:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:0.0:1784949090.936162:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000194fc0c0. 00000400:00000010:0.0:1784949090.936165:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002e7a7cb2. 00000400:00000010:0.0:1784949090.936168:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000024644ba3. 00000400:00000200:0.0:1784949090.936172:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:0.0:1784949090.936176:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:0.0:1784949090.936179:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.936182:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.936186:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.936189:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:0.0:1784949090.936193:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3610:3610:241:4294967295] 111.222.142.10@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:0.0:1784949090.936220:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:0.0:1784949090.936233:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.936240:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:0.0:1784949090.936245:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b51ad697. 00000800:00000200:0.0:1784949090.936250:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:0.1:1784949090.936258:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000400:00000200:0.0:1784949090.936266:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:0.0:1784949090.936271:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x261 rc 0 00000400:00000200:0.0:1784949090.936278:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:0.0:1784949090.936289:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004041a48e. 00000400:00000010:0.0:1784949090.936293:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000553902b5. 00000400:00000010:0.0:1784949090.936297:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000067f76b46. 00000400:00000200:0.0:1784949090.936304:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:0.0:1784949090.936314:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:0.0:1784949090.936318:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.936321:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.936325:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.936328:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:0.0:1784949090.936332:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3611:3611:240:4294967295] 111.222.173.11@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:0.0:1784949090.936338:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:0.0:1784949090.936353:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.936358:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:0.0:1784949090.936362:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f8a82730. 00000800:00000200:0.0:1784949090.936367:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:0.1:1784949090.936372:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:0.0:1784949090.936377:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:0.0:1784949090.936382:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:0.0:1784949090.936387:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:0.0:1784949090.936390:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.936393:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.936395:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f43fd4d9 00000400:00000010:0.0:1784949090.936398:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f43fd4d9. 00000400:00000200:0.0:1784949090.936403:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.936408:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:0.0:1784949090.936412:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000053ab16eb. 00000400:00000200:0.0:1784949090.936423:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-5 00000400:00000200:0.0:1784949090.936427:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -5 00000400:00000200:0.0:1784949090.936431:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000400:00000200:0.0:1784949090.936442:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:0.0:1784949090.936449:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000053ab16eb. 00000400:00000010:0.0:1784949090.936454:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f43fd4d9. 00000400:00000010:0.0:1784949090.936459:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000087a70a9. 00000400:00000200:0.0:1784949090.936470:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:0.0:1784949090.936478:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:0.0:1784949090.936488:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.936492:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.936495:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.936498:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:0.0:1784949090.936502:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3612:3612:239:4294967295] 111.222.142.11@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:0.0:1784949090.936507:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:0.0:1784949090.936519:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.936530:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:0.0:1784949090.936534:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000062b4a223. 00000800:00000200:0.0:1784949090.936544:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:0.1:1784949090.936553:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:0.0:1784949090.936572:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:0.0:1784949090.936575:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:0.0:1784949090.936579:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:0.0:1784949090.936584:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eb983bef. 00000400:00000010:0.0:1784949090.936587:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000400:00000010:0.0:1784949090.936591:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f9462ca3. 00000400:00000200:0.0:1784949090.936595:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:0.0:1784949090.936610:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:0.0:1784949090.936614:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.936618:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.936621:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.936627:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:0.0:1784949090.936637:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3613:3613:238:4294967295] 111.222.173.12@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:0.0:1784949090.936643:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:0.0:1784949090.936650:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.936656:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:0.0:1784949090.936661:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000034aacccc. 00000800:00000200:0.0:1784949090.936673:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:0.0:1784949090.936676:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784949090.936682:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:0.1:1784949090.936685:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1784949090.936689:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.12@tcp 00000800:00000010:0.0:1784949090.936691:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000034aacccc. 00000400:00000200:0.0:1784949090.938370:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784949090.938377:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784949090.938380:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949090.938384:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:0.0:1784949090.938391:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:0.0:1784949090.938399:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:0.0:1784949090.938402:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 rc 0 00000400:00000200:0.0:1784949090.938406:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:0.0:1784949090.938411:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e120de1c. 00000400:00000010:0.0:1784949090.938414:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000400:00000010:0.0:1784949090.938419:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000fd8e815. 00000400:00000200:0.0:1784949090.938424:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:0.0:1784949090.938430:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:0.0:1784949090.938434:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.938437:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.938440:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.938443:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:0.0:1784949090.938446:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3614:3614:238:4294967295] 111.222.173.13@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:0.0:1784949090.938452:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:0.0:1784949090.938463:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.938474:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:0.0:1784949090.938478:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000034aacccc. 00000800:00000200:0.0:1784949090.938484:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:0.1:1784949090.938490:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:0.0:1784949090.938494:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:0.0:1784949090.938496:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:0.0:1784949090.938499:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:0.0:1784949090.938504:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000be26f34f. 00000400:00000010:0.0:1784949090.938506:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000400:00000010:0.0:1784949090.938510:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002e599554. 00000400:00000200:0.0:1784949090.938513:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:0.0:1784949090.938517:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:0.0:1784949090.938520:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.938524:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.938527:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.938530:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:0.0:1784949090.938533:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3615:3615:237:4294967295] 111.222.142.13@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:0.0:1784949090.938538:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:0.0:1784949090.938544:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.938548:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:0.0:1784949090.938550:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d104522b. 00000800:00000200:0.0:1784949090.938554:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:0.1:1784949090.938558:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:0.0:1784949090.938577:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:0.0:1784949090.938579:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:0.0:1784949090.938582:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:0.0:1784949090.938586:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006841197d. 00000400:00000010:0.0:1784949090.938589:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000400:00000010:0.0:1784949090.938594:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c12c1931. 00000400:00000200:0.0:1784949090.938598:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:0.0:1784949090.938604:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:0.0:1784949090.938608:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.938611:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.938618:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.938624:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:0.0:1784949090.938627:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3616:3616:236:4294967295] 111.222.173.14@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:0.0:1784949090.938634:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:0.0:1784949090.938647:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.938652:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:0.0:1784949090.938656:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008f5595bf. 00000800:00000200:0.0:1784949090.938662:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:0.0:1784949090.938665:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.14 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1784949090.938672:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:0.1:1784949090.938675:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.14 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1784949090.938688:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.14@tcp 00000800:00000010:0.0:1784949090.938692:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008f5595bf. 00000400:00000200:0.0:1784949090.938698:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784949090.938703:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: REMOTE_ERROR 00000400:00000200:0.0:1784949090.938706:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949090.938713:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:0.0:1784949090.938716:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:0.0:1784949090.938724:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:0.0:1784949090.938728:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 rc 0 00000400:00000200:0.0:1784949090.938732:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:0.0:1784949090.938737:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000da755428. 00000400:00000010:0.0:1784949090.938741:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000400:00000010:0.0:1784949090.938750:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e42cbe57. 00000400:00000200:0.0:1784949090.938753:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:0.0:1784949090.938760:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:0.0:1784949090.938763:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.938767:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.938769:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.938772:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:0.0:1784949090.938775:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3617:3617:236:4294967295] 111.222.142.14@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:0.0:1784949090.938779:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:0.0:1784949090.938785:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.938789:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:0.0:1784949090.938792:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008f5595bf. 00000800:00000200:0.0:1784949090.938796:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:0.1:1784949090.938801:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:0.0:1784949090.938805:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:0.0:1784949090.938807:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:0.0:1784949090.938818:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:0.0:1784949090.938825:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f0e4e38. 00000400:00000010:0.0:1784949090.938832:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000400:00000010:0.0:1784949090.938839:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000390dd32d. 00000400:00000200:0.0:1784949090.938843:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:0.0:1784949090.938851:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:0.0:1784949090.938858:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.938863:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.938866:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.938869:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:0.0:1784949090.938873:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3618:3618:235:4294967295] 111.222.173.15@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:0.0:1784949090.938877:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:0.0:1784949090.938886:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.938893:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:0.0:1784949090.938898:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000353a5c12. 00000800:00000200:0.0:1784949090.938903:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:0.1:1784949090.938935:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:0.0:1784949090.938954:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:0.0:1784949090.938959:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:0.0:1784949090.938963:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:0.0:1784949090.938973:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b105ef1a. 00000400:00000010:0.0:1784949090.938977:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000400:00000010:0.0:1784949090.938982:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c732a1a8. 00000400:00000200:0.0:1784949090.938986:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:0.0:1784949090.938993:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:0.0:1784949090.938998:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949090.939004:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.939009:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949090.939013:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:0.0:1784949090.939021:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3619:3619:234:4294967295] 111.222.142.15@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:0.0:1784949090.939028:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:0.0:1784949090.939041:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.939047:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:0.0:1784949090.939050:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003ed79484. 00000800:00000200:0.0:1784949090.939062:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:0.1:1784949090.939069:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:0.0:1784949090.939081:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:0.0:1784949090.939087:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:0.0:1784949090.939094:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.28@tcp(00000000e7299d92) state 0x40 00000400:00000010:0.0:1784949090.939105:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000038ac69a2. 00000400:00000010:0.0:1784949090.939112:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000400:00000010:0.0:1784949090.939122:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009f5e55af. 00000400:00000200:0.0:1784949090.939133:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.28@tcp 00000400:00000200:0.0:1784949090.939140:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.142.28@tcp local destination 00000400:00000200:0.0:1784949090.939146:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949090.939151:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.142.28@tcp 00000400:00000200:0.0:1784949090.939161:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.142.28@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:0.0:1784949090.939166:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3620:3620:233:4294967295] 111.222.142.28@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:0.0:1784949090.939172:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.28@tcp 00000400:00000200:0.0:1784949090.939183:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.28@tcp(111.222.142.28@tcp:111.222.142.28@tcp) : GET try# 0 00000800:00000200:0.0:1784949090.939194:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.28@tcp 00000800:00000010:0.0:1784949090.939219:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009ff40a64. 00000800:00000010:0.0:1784949090.939244:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000be5664ea. 00000800:00000010:0.0:1784949090.939253:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 0000000026a1c0e3. 00000800:00000200:0.0:1784949090.939259:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000be5664ea] -> 12345-111.222.142.28@tcp (2) 00000800:00000200:0.1:1784949090.939263:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000be5664ea] -> 12345-111.222.142.28@tcp (2) 00000400:00000200:0.0:1784949090.939272:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.28@tcp 00000400:00000200:0.0:1784949090.939275:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.28@tcp(00000000e7299d92) state 0x240 rc 0 00000400:00000200:0.0:1784949090.939279:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:0.0:1784949090.939282:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.939284:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.939286:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:0.0:1784949090.939288:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000400:00000200:0.0:1784949090.939292:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.939295:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:0.0:1784949090.939299:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001df3b40e. 00000400:00000200:0.0:1784949090.939304:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-5 00000400:00000200:0.0:1784949090.939307:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -5 00000400:00000200:0.0:1784949090.939314:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:0.0:1784949090.939322:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000400:00000001:0.0:1784949090.939326:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.939329:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.939331:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:0.0:1784949090.939334:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000400:00000200:0.0:1784949090.939340:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.939344:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:0.0:1784949090.939349:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002fbb6924. 00000400:00000200:0.0:1784949090.939354:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-5 00000400:00000200:0.0:1784949090.939360:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -5 00000400:00000200:0.0:1784949090.939365:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:0.0:1784949090.939377:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000001:0.0:1784949090.939380:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.939381:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.939383:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:0.0:1784949090.939385:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000400:00000200:0.0:1784949090.939387:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.939390:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:0.0:1784949090.939392:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000047e19e91. 00000400:00000200:0.0:1784949090.939396:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-5 00000400:00000200:0.0:1784949090.939398:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -5 00000400:00000200:0.0:1784949090.939402:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000400:00000200:0.0:1784949090.939405:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000001:0.0:1784949090.939408:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.939409:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.939410:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:0.0:1784949090.939412:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000400:00000200:0.0:1784949090.939427:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.939431:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:0.0:1784949090.939433:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef8e57c8. 00000400:00000200:0.0:1784949090.939440:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-5 00000400:00000200:0.0:1784949090.939443:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -5 00000400:00000200:0.0:1784949090.939448:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00000400:00000200:0.0:1784949090.939451:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:0.0:1784949090.939453:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.939461:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.939463:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000400:00000010:0.0:1784949090.939466:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000400:00000200:0.0:1784949090.939468:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.939472:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:0.0:1784949090.939475:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eb983bef. 00000400:00000200:0.0:1784949090.939481:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-5 00000400:00000200:0.0:1784949090.939485:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -5 00000400:00000200:0.0:1784949090.939490:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000400:00000200:0.0:1784949090.939493:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000001:0.0:1784949090.939495:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949090.939497:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949090.939498:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:0.0:1784949090.939505:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000400:00000200:0.0:1784949090.939507:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949090.939510:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:0.0:1784949090.939514:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006841197d. 00000400:00000200:0.0:1784949090.939517:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-5 00000400:00000200:0.0:1784949090.939524:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -5 00000400:00000200:0.0:1784949090.939533:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00010000:00000001:2.0:1784949091.571967:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949091.571974:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949091.571980:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949091.571983:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949091.571985:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949091.571988:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949091.571991:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949091.571993:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949091.571996:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949091.571998:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949091.572063:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949091.572073:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949091.572079:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949091.764067:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949091.764085:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949091.764087:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949091.764094:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949091.764098:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949091.764104:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.764108:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949091.764160:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949091.764172:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949091.764190:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003492edb3. 00000020:00000040:3.0:1784949091.764231:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949091.764242:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949091.764248:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949091.764260:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949091.764278:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949091.764292:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949091.764311:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949091.764330:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949091.764334:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949091.764342:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003492edb3] to pc [ptlrpcd_00_01+1] req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949091.764388:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949091.764392:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949091.764398:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949091.764403:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949091.764408:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949091.764413:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000020:00000040:3.0:1784949091.764417:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949091.764421:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949091.764423:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784949091.764427:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949091.764427:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.764435:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1784949091.764437:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:0.0:1784949091.764440:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949091.764445:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949091.764452:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949091.764459:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949091.764500:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949091.764504:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949091.764506:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949091.764507:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.764521:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949091.764524:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949091.764526:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949091.764533:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949091.764534:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_00_02+2] req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949091.764543:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949091.764555:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289673600:0@lo:400: 00000100:00000001:0.0:1784949091.764570:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949091.764581:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949091.764586:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949091.764590:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949091.764600:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006620b0a1. 02000000:00000001:0.0:1784949091.764604:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.764608:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949091.764616:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 00000100:00000040:3.0:1784949091.764618:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11368) 00000400:00000010:0.0:1784949091.764633:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000001:2.0:1784949091.764639:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949091.764648:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289673600, portal 25 00000100:00000001:2.0:1784949091.764648:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949091.764658:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949091.764660:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949091.764673:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554958848 : -115544154592768 : ffff96e9c7b5aa00) 00000100:00000040:0.0:1784949091.764686:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949091.764691:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949091.764725:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949091.764739:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.764775:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949091.764787:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784949091.764793:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949091.764797:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949091.764800:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000100:00000200:0.0:1784949091.764811:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289673600, offset 0 02000000:00000001:2.0:1784949091.764813:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949091.764823:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949091.764835:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784949091.764837:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949091.764847:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289673728:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949091.764855:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949091.764862:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949091.764873:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949091.764874:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784949091.764888:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949091.764902:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006b8bfa56. 02000000:00000001:2.0:1784949091.764954:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949091.764963:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643af80 00000400:00000010:2.0:1784949091.764974:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000919ac10c. 00000400:00000200:0.0:1784949091.764988:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x843d [2] + 0 00000400:00000010:0.0:1784949091.764996:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b69fa69e. 00000400:00000200:0.0:1784949091.765011:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000e6a263f4 00000400:00000010:2.0:1784949091.765021:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000200:0.0:1784949091.765025:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784949091.765034:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289673728, portal 10 00000400:00000200:0.0:1784949091.765036:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949091.765058:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1784949091.765064:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6a263f4 00000100:00000001:2.0:1784949091.765064:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000400:00000010:0.0:1784949091.765081:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000e6a263f4. 00000100:00000001:0.0:1784949091.765093:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:2.0:1784949091.765095:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949091.765104:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949091.765123:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f7830c1e x1871651289673600 msgsize 224 00000100:00000001:2.0:1784949091.765145:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949091.765157:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784949091.765171:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000010:2.0:1784949091.765172:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000100:00000200:2.0:1784949091.765187:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289673728, offset 0 00000100:00000040:0.0:1784949091.765188:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949091.765226:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949091.765238:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949091.765248:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949091.765256:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949091.765262:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949091.765268:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949091.765273:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949091.765278:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3621:3621:232:4294967295] 192.168.202.137@tcp LPNI seq info [2319:2319:8:4294967295] 00000400:00000200:2.0:1784949091.765296:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949091.765305:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949091.765313:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fd30a9ad. 00000400:00000200:0.0:1784949091.765320:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000800:00000200:2.0:1784949091.765330:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:0.0:1784949091.765331:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000100:00000001:3.0:1784949091.765338:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784949091.765351:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1784949091.765355:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289673600 00000100:00000200:0.0:1784949091.765367:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949091.765367:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000200:2.0:1784949091.765369:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784949091.765381:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:2.0:1784949091.765385:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fd30a9ad type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949091.765391:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949091.765414:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.765420:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1784949091.765422:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289673600 00000020:00000001:3.0:1784949091.765431:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:0.0:1784949091.765432:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949091.765437:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949091.765437:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784949091.765449:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:2.0:1784949091.765458:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2163 usecs (3 jiffies) 00000020:00000040:3.0:1784949091.765459:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949091.765462:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949091.765483:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949091.765484:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949091.765486:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306031108 00000020:00000001:3.0:1784949091.765499:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:2.0:1784949091.765508:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000100:00000001:0.0:1784949091.765509:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.765512:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949091.765528:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784949091.765529:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306031147 00000100:00000001:2.0:1784949091.765535:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949091.765542:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949091.765549:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00000001:2.0:1784949091.765554:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306031108 00000100:00000010:2.0:1784949091.765561:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000082553b41. 00000020:00000010:3.0:1784949091.765566:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000033c25eed. 00000100:00000001:0.0:1784949091.765568:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306031108 00000100:00000001:2.0:1784949091.765570:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000010:3.0:1784949091.765587:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000d5b0a60f. 00000100:00000001:0.0:1784949091.765597:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306031147 00000100:00000001:2.0:1784949091.765600:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306031108 00000020:00000010:3.0:1784949091.765601:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949091.765614:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306031108 00000100:00000010:2.0:1784949091.765619:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000824e9543. 00000100:00000001:0.0:1784949091.765628:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306031147 00000100:00000001:2.0:1784949091.765632:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000800:00000200:1.0:1784949091.765638:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:3.0:1784949091.765642:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949091.765648:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306031108 00000800:00000010:1.0:1784949091.765651:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fd30a9ad. 00000100:00000001:0.0:1784949091.765654:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306031108 00000100:00000001:3.0:1784949091.765656:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:2.0:1784949091.765658:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000abbd7d53. 00000100:00000001:3.0:1784949091.765670:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949091.765681:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.765683:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949091.765685:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306031147 00000400:00000200:1.0:1784949091.765686:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949091.765696:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.765718:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949091.765725:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306031108 00000400:00000200:1.0:1784949091.765726:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949091.765729:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.765732:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949091.765744:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949091.765751:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949091.765757:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000100:00000001:0.0:1784949091.765759:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.765763:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949091.765769:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000010:1.0:1784949091.765775:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000100:00000001:0.0:1784949091.765777:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.765784:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949091.765789:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949091.765800:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949091.765801:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949091.765804:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1784949091.765810:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 774 00000100:00000200:1.0:1784949091.765831:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949091.765834:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784949091.765835:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949091.765836:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949091.765843:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949091.765852:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949091.765852:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782159952 : -115538927391664 : ffff96eaff466c50) 00000100:00000001:0.0:1784949091.765857:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949091.765869:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949091.765870:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.765880:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949091.765886:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f7830c1e x1871651289673600/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:752/0 lens 224/0 e 0 to 0 dl 1784949102 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949091.765890:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.765896:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949091.765965:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949091.765973:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949091.766006:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949091.766038:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949091.766055:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949091.766065:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:0.0:1784949091.766068:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949091.766069:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949091.766073:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:1.0:1784949091.766096:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949091.766096:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949091.766111:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000400:00000200:0.0:1784949091.766113:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00100000:3.0:1784949091.766130:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f7830c1e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289673600:12345-0@lo:400: 00000100:00000200:3.0:1784949091.766138:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289673600 00000400:00000200:0.0:1784949091.766146:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b000 00000020:00000001:3.0:1784949091.766148:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949091.766157:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949091.766162:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949091.766171:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 169040 00000020:00000001:3.0:1784949091.766171:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949091.766177:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949091.766188:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000001:0.0:1784949091.766214:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949091.766217:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949091.766226:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949091.766230:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:0.0:1784949091.766234:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949091.766236:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949091.766245:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784949091.766247:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949091.766259:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:0.0:1784949091.766261:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784949091.766267:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949091.766276:0:337114:0:(events.c:286:request_in_callback()) Process entered 02000000:00000010:3.0:1784949091.766280:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000980961c7. 00000100:00000200:0.0:1784949091.766281:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949091.766292:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000061b68cb4. 02000000:00000001:3.0:1784949091.766292:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949091.766304:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000061b68cb4 x1871651289673728 msgsize 224 00000100:00000001:3.0:1784949091.766306:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949091.766317:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:3.0:1784949091.766319:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949091.766328:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289673600 00010000:00000001:3.0:1784949091.766334:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949091.766345:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f7830c1e x1871651289673600/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:752/0 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949091.766346:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784949091.766360:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949091.766369:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949091.766373:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949091.766381:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949091.766390:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=113 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949091.766404:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949091.766411:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949091.766413:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949091.766416:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00100000:2.0:1784949091.766420:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289673728 02000000:00000001:3.0:1784949091.766437:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949091.766438:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:3.0:1784949091.766451:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.766452:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949091.766458:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.766464:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784949091.766466:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949091.766475:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289673728 00000100:00000040:3.0:1784949091.766477:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:2.0:1784949091.766486:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949091.766490:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:3.0:1784949091.766494:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000020:00000001:2.0:1784949091.766495:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949091.766502:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:3.0:1784949091.766508:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289673600, offset 224 00000020:00000001:2.0:1784949091.766509:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949091.766523:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949091.766532:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949091.766540:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:2.0:1784949091.766543:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000100:00000001:2.0:1784949091.766549:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949091.766554:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000400:00000200:3.0:1784949091.766565:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:2.0:1784949091.766572:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008fae13bf. 00000020:00000010:2.0:1784949091.766582:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bfe285f8. 00000400:00000200:3.0:1784949091.766591:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643af80 00000020:00000010:2.0:1784949091.766596:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000040:2.0:1784949091.766618:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:3.0:1784949091.766620:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8e45 [1] + 224 00000100:00000001:2.0:1784949091.766624:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949091.766630:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949091.766641:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949091.766645:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949091.766652:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:3.0:1784949091.766665:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949091.766666:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949091.766678:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.766686:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949091.766695:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949091.766699:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949091.766719:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949091.766724:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:3.0:1784949091.766727:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949091.766739:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 750 00000100:00000040:2.0:1784949091.766758:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949091.766769:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554974336 : -115544154577280 : ffff96e9c7b5e680) 00000100:00000040:2.0:1784949091.766792:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000061b68cb4 x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:752/0 lens 224/0 e 0 to 0 dl 1784949102 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949091.766811:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949091.766816:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949091.766825:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000400:00000200:3.0:1784949091.766828:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000100:00000001:2.0:1784949091.766832:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784949091.766838:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000020:00000010:2.0:1784949091.766845:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000100:00000001:3.0:1784949091.766850:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949091.766858:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00100000:2.0:1784949091.766863:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000061b68cb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289673728:12345-192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949091.766871:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949091.766877:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289673728 00000100:00000040:3.0:1784949091.766889:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:2.0:1784949091.766894:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00010000:00000001:3.0:1784949091.766903:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949091.766961:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949091.766964:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.766969:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949091.766971:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949091.766972:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:0.0:1784949091.766983:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1784949091.766985:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949091.766988:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f7830c1e x1871651289673600/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:752/0 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949091.766997:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949091.767008:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000020:00000001:2.0:1784949091.767008:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949091.767023:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00100000:3.0:1784949091.767024:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f7830c1e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289673600:12345-0@lo:400: Request processed in 951us (1891us total) trans 0 rc 0/0 00000400:00000200:0.0:1784949091.767028:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:0.0:1784949091.767047:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000020:00000001:2.0:1784949091.767049:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949091.767067:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949091.767068:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1784949091.767082:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 774 00000020:00000001:2.0:1784949091.767085:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:0.0:1784949091.767086:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949091.767090:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:2.0:1784949091.767097:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949091.767106:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:2.0:1784949091.767108:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949091.767115:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:2.0:1784949091.767128:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000010:3.0:1784949091.767133:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000d5b0a60f. 00000100:00000001:2.0:1784949091.767148:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000010:3.0:1784949091.767149:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000200:0.0:1784949091.767155:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949091.767159:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:3.0:1784949091.767164:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000033c25eed. 02000000:00000010:2.0:1784949091.767168:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000de31af8d. 00000020:00000040:3.0:1784949091.767177:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:2.0:1784949091.767178:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.767216:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949091.767223:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:2.0:1784949091.767232:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949091.767233:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000980961c7. 00000100:00000001:0.0:1784949091.767239:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000040:2.0:1784949091.767250:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289673728 02000000:00000001:3.0:1784949091.767252:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949091.767253:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949091.767260:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949091.767263:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949091.767274:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:3.0:1784949091.767278:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000200:2.0:1784949091.767284:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000061b68cb4 x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:752/0 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949091.767289:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:3.0:1784949091.767295:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001004a12d. 00000100:00000001:0.0:1784949091.767303:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949091.767313:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949091.767316:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000092ad473a. 02000000:00000001:0.0:1784949091.767329:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949091.767352:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00001000:0.0:1784949091.767353:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=113 v=5 (5 5 5 5) 00010000:00000001:2.0:1784949091.767359:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949091.767363:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=113 v=5 (1 1 1 1) 00000100:00001000:2.0:1784949091.767366:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=108 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949091.767373:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949091.767379:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949091.767380:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949091.767392:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000040:2.0:1784949091.767398:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949091.767401:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.767404:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784949091.767407:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949091.767411:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949091.767415:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949091.767420:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949091.767423:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949091.767450:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949091.767470:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000100:00000001:0.0:1784949091.767472:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949091.767489:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000200:2.0:1784949091.767491:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289673728, offset 224 00000100:00000001:0.0:1784949091.767494:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949091.767511:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949091.767513:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784949091.767535:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289673600:0@lo:400: 00000400:00000200:2.0:1784949091.767548:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949091.767566:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949091.767577:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949091.767577:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3622:3622:232:4294967295] 192.168.202.137@tcp LPNI seq info [2320:2320:8:4294967295] 00000100:00000040:0.0:1784949091.767595:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003492edb3 x1871651289673600/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949091.767639:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949091.767641:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949091.767649:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:2.0:1784949091.767661:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784949091.767663:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006620b0a1. 02000000:00000001:0.0:1784949091.767690:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000010:2.0:1784949091.767690:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005c3de731. 00000020:00000001:0.0:1784949091.767699:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949091.767704:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000800:00000200:2.0:1784949091.767719:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:0.0:1784949091.767721:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949091.767730:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 00000800:00000200:2.0:1784949091.767739:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000001:0.0:1784949091.767741:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949091.767754:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:2.0:1784949091.767754:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005c3de731 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:0.0:1784949091.767764:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003492edb3. 00000100:00000001:0.0:1784949091.767772:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949091.767775:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949091.767779:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949091.767785:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949091.767791:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.767793:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949091.767821:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949091.767823:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949091.767826:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949091.767829:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949091.767837:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949091.767841:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949091.767847:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000061b68cb4 x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:752/0 lens 224/224 e 0 to 0 dl 1784949102 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949091.767861:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000061b68cb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289673728:12345-192.168.202.137@tcp:400: Request processed in 1046us (1554us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949091.767875:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 750 00000100:00000040:2.0:1784949091.767881:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949091.767887:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949091.767893:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949091.767902:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bfe285f8. 00000020:00000010:2.0:1784949091.767951:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949091.767959:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000008fae13bf. 00000020:00000040:2.0:1784949091.767977:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949091.767984:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949091.768113:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949091.768121:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005c3de731. 00000400:00000200:3.0:1784949091.768128:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949091.768136:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949091.768143:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:3.0:1784949091.768145:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000100:00000001:3.0:1784949091.768150:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949091.768153:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949091.768161:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949091.768170:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949091.768174:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949091.768181:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949091.768187:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949091.768236:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b000 00000400:00000200:3.0:1784949091.768248:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8e4d [1] + 224 00000800:00000001:3.0:1784949091.768256:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949091.768274:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949091.768277:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949091.768282:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949091.768287:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949091.768293:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949091.768306:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949091.768329:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949091.768336:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949091.768339:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949091.768436:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949091.768443:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949091.768452:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949091.768455:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949091.768459:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000400:00000200:2.0:1784949091.768467:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:2.0:1784949091.768470:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000100:00000001:2.0:1784949091.768475:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949091.768484:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949091.768497:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949091.768508:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949091.768510:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949091.768513:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949091.768517:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949091.768519:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949091.768522:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949091.768525:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949091.768532:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=108 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949091.768536:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=108 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949091.768540:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949091.768542:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949091.768545:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949091.768550:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949091.768560:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.768564:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949091.768566:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949091.768568:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949091.768570:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949091.768574:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949091.768582:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949091.768586:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949091.768588:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949091.768591:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949091.768601:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289673728:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949091.768607:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949091.768609:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949091.768612:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289673728/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949091.768620:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949091.768623:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949091.768626:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006b8bfa56. 02000000:00000001:2.0:1784949091.768635:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949091.768637:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949091.768639:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949091.768642:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949091.768645:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:2.0:1784949091.768649:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949091.768651:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949091.768653:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000100:00000001:2.0:1784949091.768670:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949091.768671:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949091.768674:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949091.768680:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949091.768681:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949091.768684:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949091.768687:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949091.808045:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306031147, at 4306031148 00000100:00000040:2.0:1784949091.808057:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306031147, at 4306031148 00000100:00000040:3.0:1784949091.808057:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306031147, at 4306031148 00000100:00000010:1.0:1784949091.808063:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000100:00000040:0.0:1784949091.808072:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306031147, at 4306031148 00000100:00000010:3.0:1784949091.808085:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000abbd7d53. 00000100:00000010:2.0:1784949091.808090:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000824e9543. 00000100:00000010:0.0:1784949091.808101:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000800:00000100:1.0:1784949091.956240:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000800:00000010:1.0:1784949091.956251:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000400:00000200:1.0:1784949091.956264:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949091.956275:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949091.956450:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11364 health :900 00000400:00000200:1.0:1784949091.956484:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949091.956490:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:1.0:1784949091.956496:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:0.0:1784949091.956583:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949091.956599:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2861 00000400:00000001:0.0:1784949091.956609:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949091.956612:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949091.956615:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007d821c01 00000400:00000010:0.0:1784949091.956619:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007d821c01. 00000400:00000200:0.0:1784949091.956626:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949091.956633:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:0.0:1784949091.956639:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf7eadde. 00000400:00000200:0.0:1784949091.956646:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:0.0:1784949091.956650:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:0.0:1784949091.956654:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00010000:00000001:2.0:1784949092.596039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949092.596049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949092.596057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949092.596058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949092.596147:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949092.596155:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949092.596159:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949092.788035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949092.788056:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949092.788059:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949092.788066:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949092.788071:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949092.788076:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949092.788079:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949092.980081:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:1.0:1784949092.980086:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.8@tcp 00000400:00000010:3.0:1784949092.980093:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000044ae300f. 00000800:00000010:1.0:1784949092.980096:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b9b32818. 00000400:00000010:3.0:1784949092.980114:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 00000400:00000200:1.0:1784949092.980116:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949092.980139:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000400:00000200:1.0:1784949092.980149:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949092.980164:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11366 health :800 00000400:00000010:3.0:1784949092.980171:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004fd5390b. 00000400:00000200:1.0:1784949092.980172:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949092.980219:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:3.0:1784949092.980232:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949092.980238:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:3.0:1784949092.980290:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949092.980302:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949092.980310:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3623:3623:233:4294967295] 192.168.202.137@tcp LPNI seq info [2321:2321:8:4294967295] 00000400:00000200:3.0:1784949092.980322:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949092.980333:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949092.980341:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949092.980349:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000400:00000200:0.0:1784949092.980354:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784949092.980364:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949092.980371:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2861 00000800:00000200:3.0:1784949092.980384:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:0.0:1784949092.980385:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949092.980390:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:3.0:1784949092.980396:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1784949092.980397:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000400:00000010:0.0:1784949092.980405:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000400:00000200:0.0:1784949092.980414:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949092.980423:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:0.0:1784949092.980429:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f0621080. 00000400:00000200:0.0:1784949092.980439:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-110 00000400:00000200:0.0:1784949092.980448:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -110 00000800:00000100:1.0:1784949092.980458:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000400:00000200:0.0:1784949092.980461:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000800:00000010:1.0:1784949092.980474:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ca1124f5. 00000400:00000200:1.0:1784949092.980484:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949092.980500:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949092.980510:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949092.980515:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:1.0:1784949092.980520:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000800:00000100:1.0:1784949092.980552:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:1.0:1784949092.980556:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000200:1.0:1784949092.980562:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949092.980571:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949092.980578:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949092.980586:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949092.980588:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:0.0:1784949092.980598:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2861 00000400:00000200:1.0:1784949092.980606:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000001:0.0:1784949092.980609:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949092.980613:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949092.980620:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:0.0:1784949092.980624:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000400:00000200:0.0:1784949092.980629:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949092.980647:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:0.0:1784949092.980655:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dd0652fb. 00000400:00000200:0.0:1784949092.980663:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:0.0:1784949092.980671:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:0.0:1784949092.980676:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000800:00000200:2.0:1784949092.980685:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1784949092.980686:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2861 00000400:00000001:0.0:1784949092.980699:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000010:2.0:1784949092.980702:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000001:0.0:1784949092.980704:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949092.980711:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:0.0:1784949092.980719:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000800:00000001:3.0:1784949092.980721:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949092.980724:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949092.980725:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949092.980732:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:0.0:1784949092.980740:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001a4eba99. 00000800:00000001:3.0:1784949092.980745:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949092.980755:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000800:00000001:3.0:1784949092.980759:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949092.980760:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949092.980767:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000800:00000001:3.0:1784949092.980777:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949092.980787:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949092.980788:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000400:00000200:2.0:1784949092.980805:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:3.0:1784949092.980814:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949092.980844:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949092.980859:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949092.980872:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949092.980879:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3624:3624:236:4294967295] 192.168.202.137@tcp LPNI seq info [2322:2322:8:4294967295] 00000400:00000200:3.0:1784949092.980889:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949092.980895:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949092.980900:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949092.980938:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949092.980943:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949092.980947:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949092.981009:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949092.981127:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949092.981136:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:3.0:1784949092.981142:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949092.981149:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949092.981151:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949092.981171:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949092.981180:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949092.981207:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949092.981210:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949092.981229:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949092.981256:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8e61 00000800:00000001:2.0:1784949092.981266:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949092.981285:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949092.981289:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949092.981293:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949092.981302:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000400:00000010:2.0:1784949092.981305:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000400:00000200:2.0:1784949092.981309:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949092.981313:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000044ae300f. 00000400:00000010:2.0:1784949092.981317:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 00000800:00000001:2.0:1784949092.981326:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949092.981329:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949093.620074:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949093.620084:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949093.620093:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949093.620094:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949093.620233:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949093.620242:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949093.620247:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949093.812073:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949093.812093:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949093.812096:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949093.812102:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949093.812107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949093.812112:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949093.812115:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949094.644094:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949094.644111:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949094.644121:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949094.644123:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949094.644243:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949094.644248:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949094.644252:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949094.836081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949094.837888:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949094.837894:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949094.837957:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949094.837963:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949094.837984:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949094.837989:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949095.668157:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949095.668206:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949095.668215:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949095.668217:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949095.668326:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949095.670078:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949095.670085:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949095.864005:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949095.865804:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949095.865809:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949095.865816:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949095.865821:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949095.865826:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949095.865829:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949096.692077:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949096.692087:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949096.692096:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949096.692099:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949096.692257:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949096.692268:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949096.692274:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784949096.884107:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949096.884129:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784949096.884207:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949096.884224:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949096.884227:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949096.884234:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949096.884238:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949096.884243:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949096.884247:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949096.886496:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008c05042f. 00000020:00000040:1.0:1784949096.886513:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949096.886521:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949096.886526:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949096.886534:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949096.886546:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000004b264730. 00000100:00000001:1.0:1784949096.886555:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949096.886567:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949096.886582:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949096.886586:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949096.886593:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008c05042f] to pc [ptlrpcd_00_03+3] req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949096.886635:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949096.886640:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949096.886645:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949096.886648:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949096.886653:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949096.886657:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ee628dd6. 00000020:00000040:1.0:1784949096.886660:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949096.886663:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949096.886665:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949096.886667:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949096.886670:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fee8262a. 00000100:00000001:1.0:1784949096.886673:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949096.886679:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949096.886689:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949096.886691:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949096.886694:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee628dd6] to pc [ptlrpcd_00_00+0] req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.886750:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949096.886752:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.886755:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949096.886763:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:0.0:1784949096.886766:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.886769:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.886785:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949096.886787:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11373) 00000100:00000001:2.0:1784949096.886789:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.886798:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949096.886802:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949096.886807:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949096.886813:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949096.886818:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949096.886819:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949096.886821:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:3.0:1784949096.886835:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.886838:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949096.886859:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949096.886862:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949096.886864:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.886867:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.886873:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949096.886875:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949096.886877:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1784949096.886882:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949096.886883:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1784949096.886893:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949096.886902:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008c05042f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289673856:0@lo:400: 00000100:00000040:0.0:1784949096.886947:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.886991:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949096.886994:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949096.886998:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949096.887001:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949096.887004:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949096.887006:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949096.887011:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949096.887013:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1784949096.887026:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949096.887031:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000008fae13bf. 00000100:00100000:0.0:1784949096.887038:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ee628dd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289673984:192.168.202.137@tcp:400: 02000000:00000001:2.0:1784949096.887040:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.887061:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:2.0:1784949096.887064:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000919ac10c. 02000000:00000001:0.0:1784949096.887068:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949096.887074:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949096.887077:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 02000000:00000001:0.0:1784949096.887079:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949096.887100:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006620b0a1. 00000100:00000200:2.0:1784949096.887102:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289673856, portal 25 02000000:00000001:0.0:1784949096.887125:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.887125:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949096.887136:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637808256 : -115544071743360 : ffff96e9cca5d880) 00000400:00000010:0.0:1784949096.887141:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 00000100:00000040:2.0:1784949096.887176:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949096.887186:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000001:2.0:1784949096.887206:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:0.0:1784949096.887213:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289673984, portal 10 00000100:00000040:2.0:1784949096.887216:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949096.887227:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:2.0:1784949096.887229:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000100:00000001:0.0:1784949096.887235:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637816320 : -115544071735296 : ffff96e9cca5f800) 00000100:00000200:2.0:1784949096.887238:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289673856, offset 0 00000100:00000040:0.0:1784949096.887254:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949096.887259:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949096.887282:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949096.887288:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949096.887300:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000400:00000200:2.0:1784949096.887302:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:0.0:1784949096.887324:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289673984, offset 0 00000400:00000200:0.0:1784949096.887336:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949096.887368:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b080 00000400:00000200:0.0:1784949096.887383:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949096.887385:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x845d [2] + 0 00000400:00000010:2.0:1784949096.887404:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fed0cd52. 00000400:00000200:0.0:1784949096.887405:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949096.887422:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949096.887425:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000047756a57 00000400:00000200:0.0:1784949096.887433:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949096.887438:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949096.887444:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949096.887445:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3625:3625:236:4294967295] 192.168.202.137@tcp LPNI seq info [2323:2323:8:4294967295] 00000400:00000200:2.0:1784949096.887466:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949096.887479:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949096.887489:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949096.887491:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000047756a57 00000400:00000010:2.0:1784949096.887500:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000047756a57. 00000800:00000010:0.0:1784949096.887501:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003a8a98bc. 00000100:00000001:2.0:1784949096.887508:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949096.887513:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949096.887517:0:337121:0:(events.c:337:request_in_callback()) incoming req@000000002b339182 x1871651289673856 msgsize 224 00000100:00100000:2.0:1784949096.887524:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949096.887526:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949096.887542:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949096.887554:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:2.0:1784949096.887562:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000800:00000200:0.0:1784949096.887564:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949096.887572:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949096.887579:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.887592:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949096.887596:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.887605:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.887608:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949096.887614:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.887620:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306036269 00000100:00000010:2.0:1784949096.887627:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000824e9543. 00000100:00000001:2.0:1784949096.887637:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949096.887647:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306036269 00000100:00000010:2.0:1784949096.887651:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 00000100:00000001:2.0:1784949096.887657:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949096.887664:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306036269 00000100:00000001:1.0:1784949096.887665:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000010:2.0:1784949096.887667:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f59314ac. 00000100:00100000:1.0:1784949096.887672:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289673856 00000100:00000001:2.0:1784949096.887675:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:1.0:1784949096.887678:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949096.887683:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306036269 00000100:00000001:1.0:1784949096.887684:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949096.887692:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949096.887693:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f99e6ef2. 00000100:00000001:2.0:1784949096.887701:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:1.0:1784949096.887702:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949096.887710:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289673856 00000020:00000001:1.0:1784949096.887713:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949096.887714:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1784949096.887716:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949096.887725:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949096.887726:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949096.887731:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949096.887733:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949096.887741:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949096.887744:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949096.887748:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949096.887759:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949096.887766:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:1.0:1784949096.887773:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949096.887779:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000020:00000010:1.0:1784949096.887792:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004520da55. 00000020:00000010:1.0:1784949096.887800:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000169a7180. 00000020:00000010:1.0:1784949096.887806:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949096.887817:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949096.887822:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949096.887824:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949096.887826:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949096.887836:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949096.887856:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949096.887868:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949096.887871:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949096.887877:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 775 00000100:00000040:1.0:1784949096.887882:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949096.887885:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782154832 : -115538927396784 : ffff96eaff465850) 00000100:00000040:1.0:1784949096.887893:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000002b339182 x1871651289673856/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:2/0 lens 224/0 e 0 to 0 dl 1784949107 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949096.887946:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949096.887950:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:1.0:1784949096.887955:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949096.887961:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00100000:1.0:1784949096.887967:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000002b339182 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289673856:12345-0@lo:400: 00000100:00000200:1.0:1784949096.887973:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289673856 00000020:00000001:1.0:1784949096.887978:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949096.887983:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949096.887986:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949096.887989:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949096.887991:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949096.887995:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949096.887997:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949096.888000:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949096.888002:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949096.888005:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949096.888008:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949096.888011:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949096.888014:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949096.888019:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000095d8d756. 02000000:00000001:1.0:1784949096.888022:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949096.888024:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949096.888027:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949096.888030:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289673856 00010000:00000001:1.0:1784949096.888034:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949096.888038:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000002b339182 x1871651289673856/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:2/0 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949096.888049:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949096.888052:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949096.888056:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=118 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949096.888061:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949096.888065:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949096.888067:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784949096.888071:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949096.888073:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949096.888077:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949096.888079:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949096.888083:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000200:1.0:1784949096.888088:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289673856, offset 224 00000400:00000200:1.0:1784949096.888094:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949096.888105:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784949096.888113:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b080 00000400:00000200:1.0:1784949096.888120:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8e65 [1] + 224 00000400:00000200:1.0:1784949096.888127:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949096.888131:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949096.888136:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949096.888140:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949096.888182:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949096.888203:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784949096.888211:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784949096.888214:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:1.0:1784949096.888217:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949096.888220:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784949096.888225:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949096.888230:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784949096.888233:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949096.888236:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949096.888239:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949096.888244:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002b339182 x1871651289673856/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:2/0 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.888246:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949096.888251:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949096.888262:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:1.0:1784949096.888266:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000002b339182 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289673856:12345-0@lo:400: Request processed in 316us (743us total) trans 0 rc 0/0 00000400:00000001:2.0:1784949096.888268:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949096.888273:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000400:00000200:2.0:1784949096.888283:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:2.0:1784949096.888289:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000100:00100000:1.0:1784949096.888295:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 775 00000100:00000001:2.0:1784949096.888297:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784949096.888302:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:2.0:1784949096.888308:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949096.888310:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949096.888315:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949096.888322:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000169a7180. 00000020:00000010:1.0:1784949096.888334:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000200:2.0:1784949096.888335:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949096.888342:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004520da55. 00000020:00000040:1.0:1784949096.888354:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784949096.888356:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1784949096.888364:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784949096.888364:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949096.888370:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000095d8d756. 00000100:00000001:2.0:1784949096.888372:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949096.888380:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:1.0:1784949096.888381:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949096.888383:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949096.888387:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.888388:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949096.888395:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949096.888396:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:2.0:1784949096.888402:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=118 v=5 (5 5 5 5) 00000400:00000010:1.0:1784949096.888404:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009a55be35. 00000100:00001000:2.0:1784949096.888415:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=118 v=5 (1 1 1 1) 00000400:00000010:1.0:1784949096.888417:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ba56b0c8. 00000100:00000001:2.0:1784949096.888423:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949096.888427:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949096.888432:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949096.888435:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.888438:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949096.888442:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.888452:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949096.888456:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949096.888458:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949096.888462:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949096.888471:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008c05042f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289673856:0@lo:400: 00000100:00000001:2.0:1784949096.888477:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.888479:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949096.888481:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008c05042f x1871651289673856/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.888488:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949096.888491:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949096.888494:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000008fae13bf. 02000000:00000001:2.0:1784949096.888500:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949096.888502:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949096.888505:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949096.888507:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949096.888510:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000004b264730. 02000000:00000001:2.0:1784949096.888514:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949096.888516:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949096.888529:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008c05042f. 00000100:00000001:2.0:1784949096.888534:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949096.888535:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.888537:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.888542:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949096.888544:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.888547:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949096.888549:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949096.889433:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949096.889443:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003a8a98bc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949096.889474:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949096.889484:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2167 usecs (3 jiffies) 00000100:00000001:0.0:1784949096.889491:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306036232 00000100:00000001:0.0:1784949096.889497:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306036269 00000100:00000001:0.0:1784949096.889502:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306036232 00000100:00000001:0.0:1784949096.889508:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306036269 00000100:00000001:0.0:1784949096.889512:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306036232 00000100:00000001:0.0:1784949096.889516:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306036269 00000100:00000001:0.0:1784949096.889521:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306036232 00000100:00000001:0.0:1784949096.889527:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306036269 00000100:00000001:0.0:1784949096.889532:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949096.889538:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.889544:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.889550:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949096.889699:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949096.889709:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000400:00000200:3.0:1784949096.889719:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949096.889730:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949096.889738:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000400:00000010:3.0:1784949096.889742:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000800:00000001:0.0:1784949096.889748:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949096.889749:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949096.889761:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949096.889776:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949096.889778:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949096.889786:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949096.889788:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949096.889798:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949096.889801:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949096.889807:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949096.889817:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949096.889843:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b100 00000400:00000200:0.0:1784949096.889859:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 169264 00000800:00000001:0.0:1784949096.889872:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949096.889882:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949096.889888:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949096.889896:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949096.889946:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949096.889951:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949096.889961:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e19c9a62. 00000100:00000040:0.0:1784949096.889968:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000e19c9a62 x1871651289673984 msgsize 224 00000100:00100000:0.0:1784949096.889978:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949096.890044:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784949096.890059:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949096.890066:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.890181:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949096.890187:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289673984 02000000:00000001:2.0:1784949096.890191:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949096.890193:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949096.890196:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949096.890200:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949096.890204:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289673984 00000020:00000001:2.0:1784949096.890207:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949096.890209:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949096.890212:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949096.890215:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949096.890219:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949096.890222:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949096.890227:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949096.890233:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:2.0:1784949096.890236:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949096.890239:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000020:00000010:2.0:1784949096.890245:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008fae13bf. 00000020:00000010:2.0:1784949096.890249:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bfe285f8. 00000020:00000010:2.0:1784949096.890254:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000040:2.0:1784949096.890262:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949096.890265:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949096.890267:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949096.890269:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.890275:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949096.890278:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949096.890280:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.890302:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949096.890313:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949096.890315:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949096.890322:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 751 00000100:00000040:2.0:1784949096.890326:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949096.890330:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554973440 : -115544154578176 : ffff96e9c7b5e300) 00000100:00000040:2.0:1784949096.890337:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e19c9a62 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:2/0 lens 224/0 e 0 to 0 dl 1784949107 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949096.890351:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949096.890355:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:2.0:1784949096.890358:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949096.890361:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00100000:2.0:1784949096.890368:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e19c9a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289673984:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784949096.890374:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289673984 00000020:00000001:2.0:1784949096.890377:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949096.890380:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949096.890383:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949096.890385:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949096.890388:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949096.890391:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949096.890394:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949096.890396:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949096.890398:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949096.890401:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949096.890403:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949096.890407:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949096.890409:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949096.890414:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a6fbdd8b. 02000000:00000001:2.0:1784949096.890416:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.890419:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949096.890421:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949096.890424:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289673984 00010000:00000001:2.0:1784949096.890427:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949096.890431:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e19c9a62 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:2/0 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949096.890441:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949096.890444:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949096.890447:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=113 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949096.890451:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949096.890455:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949096.890457:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949096.890460:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949096.890462:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949096.890464:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949096.890467:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949096.890470:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000100:00000200:2.0:1784949096.890474:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289673984, offset 224 00000400:00000200:2.0:1784949096.890480:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949096.890493:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949096.890502:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3626:3626:236:4294967295] 192.168.202.137@tcp LPNI seq info [2324:2324:8:4294967295] 00000400:00000200:2.0:1784949096.890516:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949096.890523:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949096.890529:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000313a63a1. 00000800:00000200:2.0:1784949096.890537:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949096.890548:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949096.890553:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000313a63a1 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949096.890576:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949096.890580:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949096.890583:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949096.890585:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949096.890587:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949096.890591:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e19c9a62 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:2/0 lens 224/224 e 0 to 0 dl 1784949107 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949096.890603:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e19c9a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289673984:12345-192.168.202.137@tcp:400: Request processed in 250us (632us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949096.890618:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 751 00000100:00000040:2.0:1784949096.890623:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949096.890627:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949096.890631:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949096.890636:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bfe285f8. 00000020:00000010:2.0:1784949096.890641:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949096.890646:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000008fae13bf. 00000020:00000040:2.0:1784949096.890651:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949096.890654:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949096.890784:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949096.890791:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000313a63a1. 00000400:00000200:0.0:1784949096.890800:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949096.890810:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949096.890810:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949096.890822:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:0.0:1784949096.890825:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000800:00000001:3.0:1784949096.890826:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949096.890832:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784949096.890833:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949096.890839:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949096.890841:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949096.890856:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949096.890874:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b100 00000400:00000200:3.0:1784949096.890883:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8e69 [1] + 224 00000800:00000001:3.0:1784949096.890890:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949096.890900:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949096.890933:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949096.890938:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949096.890944:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949096.890949:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949096.890961:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949096.890983:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949096.890992:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949096.890996:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949096.891034:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949096.891040:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949096.891049:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949096.891052:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949096.891055:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000400:00000200:0.0:1784949096.891061:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000400:00000010:0.0:1784949096.891063:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784949096.891066:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949096.891073:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949096.891101:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949096.891111:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949096.891113:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949096.891116:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949096.891120:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949096.891124:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949096.891126:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949096.891129:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949096.891140:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=113 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949096.891145:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=113 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949096.891171:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949096.891174:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949096.891183:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949096.891189:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949096.891200:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.891205:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949096.891207:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949096.891210:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949096.891212:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949096.891216:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949096.891225:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949096.891228:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949096.891231:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949096.891236:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949096.891246:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ee628dd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289673984:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949096.891252:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949096.891254:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949096.891257:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ee628dd6 x1871651289673984/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949112 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949096.891265:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949096.891268:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949096.891271:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006620b0a1. 02000000:00000001:0.0:1784949096.891276:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949096.891279:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949096.891282:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949096.891285:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949096.891288:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fee8262a. 02000000:00000001:0.0:1784949096.891292:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949096.891294:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949096.891296:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ee628dd6. 00000100:00000001:0.0:1784949096.891300:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949096.891302:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949096.891304:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949096.891309:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949096.891311:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949096.891313:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949096.891316:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949096.930070:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306036269, at 4306036270 00000100:00000040:3.0:1784949096.930070:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306036269, at 4306036270 00000100:00000040:0.0:1784949096.930079:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306036269, at 4306036270 00000100:00000040:2.0:1784949096.930082:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306036269, at 4306036270 00000100:00000010:1.0:1784949096.930096:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00000100:00000010:0.0:1784949096.930101:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000824e9543. 00000100:00000010:2.0:1784949096.930106:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f59314ac. 00000100:00000010:3.0:1784949096.930113:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f99e6ef2. 00010000:00000001:2.0:1784949097.716057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949097.716067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949097.716075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949097.716077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949097.716449:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949097.716455:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949097.716459:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949097.908181:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949097.909821:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949097.909825:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949097.909839:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949097.909851:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949097.909855:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949097.909858:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949098.740010:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949098.740020:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949098.740029:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949098.740032:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949098.740124:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949098.740129:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949098.740161:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949098.932037:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949098.932054:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949098.932055:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949098.932061:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949098.932064:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949098.932068:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949098.932070:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1784949099.444304:0:344010:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.14:988 00000400:00000100:3.0:1784949099.444306:0:344014:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.27:988 00000400:00000100:2.0:1784949099.444307:0:344012:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.8:988 00000400:00000100:0.0:1784949099.444325:0:343956:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.12:988 00000400:00000100:0.0:1784949099.444516:0:343956:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.12@tcp at host 111.222.142.12:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784949099.444518:0:344012:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.8@tcp at host 111.222.142.8:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784949099.444544:0:344010:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.14@tcp at host 111.222.142.14:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784949099.444546:0:344014:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.27@tcp at host 111.222.142.27:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949099.444566:0:343956:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.12@tcp: down 00000400:00000200:2.0:1784949099.444566:0:344012:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.8@tcp: down 00000400:00000200:2.0:1784949099.444585:0:344012:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:0.0:1784949099.444594:0:343956:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949099.444608:0:344010:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.14@tcp: down 00000400:00000200:3.0:1784949099.444613:0:344014:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.27@tcp: down 00000400:00000200:1.0:1784949099.444625:0:344010:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784949099.444638:0:344010:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000800:00000200:3.0:1784949099.444644:0:344014:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.142.27@tcp down 00000800:00000010:1.0:1784949099.444653:0:344010:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008f5595bf. 00000400:00000200:1.0:1784949099.444665:0:344010:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949099.444676:0:344010:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949099.444693:0:344010:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11372 health :900 00000400:00000200:1.0:1784949099.444722:0:344010:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949099.444731:0:344010:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000100:3.0:1784949099.444749:0:344015:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.6:988 00000400:00000200:1.0:1784949099.444767:0:344010:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000100:3.0:1784949099.444809:0:344015:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.6@tcp at host 111.222.142.6:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784949099.444826:0:344015:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.6@tcp: down 00000400:00000200:3.0:1784949099.444835:0:344015:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1784949099.444848:0:344015:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.6@tcp 00000400:00000100:2.0:1784949099.444850:0:343911:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.4:988 00000400:00000200:0.0:1784949099.444859:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000010:3.0:1784949099.444860:0:344015:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d1b12746. 00000400:00000100:1.0:1784949099.444865:0:343953:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.2:988 00000400:00000200:3.0:1784949099.444881:0:344015:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949099.444883:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:0.0:1784949099.444901:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949099.444976:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949099.444981:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:0.0:1784949099.444987:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000400:00000200:3.0:1784949099.444988:0:344015:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949099.445007:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949099.445016:0:344015:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949099.445025:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:0.0:1784949099.445034:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000da755428. 00000400:00000100:2.0:1784949099.445041:0:343911:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.4@tcp at host 111.222.142.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784949099.445042:0:343953:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.2@tcp at host 111.222.142.2:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949099.445072:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:2.0:1784949099.445075:0:343911:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.4@tcp: down 00000400:00000200:1.0:1784949099.445076:0:343953:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.2@tcp: down 00000400:00000200:0.0:1784949099.445086:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000400:00000200:1.0:1784949099.445114:0:343953:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:2.0:1784949099.445124:0:343911:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784949099.445126:0:343953:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.2@tcp 00000800:00000010:1.0:1784949099.445156:0:343953:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000595fa8f8. 00000400:00000200:1.0:1784949099.445166:0:343953:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949099.445171:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00000800:00000100:2.0:1784949099.445175:0:343911:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.4@tcp 00000800:00000010:2.0:1784949099.445186:0:343911:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a3cf4688. 00000400:00000200:1.0:1784949099.445188:0:343953:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949099.445202:0:343911:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949099.445203:0:343953:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949099.445214:0:343953:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:1.0:1784949099.445228:0:343953:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:2.0:1784949099.445235:0:343911:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949099.445239:0:344015:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000100:0.0:1784949099.445246:0:344008:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.10:988 00000400:00000200:3.0:1784949099.445273:0:344015:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:2.0:1784949099.445298:0:343911:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949099.445314:0:343911:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:2.0:1784949099.445331:0:343911:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:1.0:1784949099.445350:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000100:0.0:1784949099.445351:0:344008:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.10@tcp at host 111.222.142.10:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949099.445369:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000200:0.0:1784949099.445374:0:344008:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.10@tcp: down 00000400:00000001:1.0:1784949099.445379:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:0.0:1784949099.445382:0:344008:0:(router.c:1888:lnet_notify()) Old news 00000400:00000001:1.0:1784949099.445385:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949099.445390:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000800:00000100:0.0:1784949099.445396:0:344008:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.10@tcp 00000400:00000010:1.0:1784949099.445397:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000800:00000010:0.0:1784949099.445406:0:344008:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b51ad697. 00000400:00000200:1.0:1784949099.445409:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949099.445415:0:344008:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949099.445420:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:1.0:1784949099.445428:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fff5e130. 00000400:00000200:0.0:1784949099.445429:0:344008:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949099.445449:0:344008:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949099.445451:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-110 00000400:00000200:0.0:1784949099.445458:0:344008:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:1.0:1784949099.445462:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -110 00000400:00000200:0.0:1784949099.445475:0:344008:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:1.0:1784949099.445488:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:1.0:1784949099.445503:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 00000400:00000001:1.0:1784949099.445508:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949099.445510:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949099.445512:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000400:00000010:1.0:1784949099.445515:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000400:00000200:1.0:1784949099.445520:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949099.445523:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:1.0:1784949099.445527:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f11303f8. 00000400:00000200:1.0:1784949099.445530:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-110 00000400:00000200:1.0:1784949099.445533:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -110 00000400:00000200:1.0:1784949099.445537:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:1.0:1784949099.445541:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:1.0:1784949099.445543:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949099.445545:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949099.445547:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000400:00000010:1.0:1784949099.445549:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000400:00000200:1.0:1784949099.445551:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949099.445553:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:1.0:1784949099.445556:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003cdea007. 00000400:00000200:1.0:1784949099.445559:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-110 00000400:00000200:1.0:1784949099.445562:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -110 00000400:00000200:1.0:1784949099.445564:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:1.0:1784949099.445567:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:1.0:1784949099.445570:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949099.445571:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949099.445572:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e7a7cb2 00000400:00000010:1.0:1784949099.445574:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002e7a7cb2. 00000400:00000200:1.0:1784949099.445576:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949099.445579:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:1.0:1784949099.445581:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000194fc0c0. 00000400:00000200:1.0:1784949099.445585:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-110 00000400:00000200:1.0:1784949099.445587:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -110 00000400:00000200:1.0:1784949099.445590:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00010000:00000001:2.0:1784949099.764016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949099.764026:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949099.764068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949099.764070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949099.764228:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949099.764238:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949099.764244:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949099.957403:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949099.958558:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949099.958561:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949099.958569:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949099.958573:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949099.959831:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949099.959836:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949100.148067:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949100.148084:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000177dc44a. 00000400:00000010:3.0:1784949100.148099:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 00000400:00000010:3.0:1784949100.148106:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000400:00000010:3.0:1784949100.148116:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008c288862. 00000400:00000200:3.0:1784949100.148120:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949100.148152:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949100.148158:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949100.148162:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3627:3627:241:4294967295] 192.168.202.137@tcp LPNI seq info [2325:2325:8:4294967295] 00000400:00000200:3.0:1784949100.148169:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949100.148176:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949100.148182:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949100.148187:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000800:00000200:3.0:1784949100.148195:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949100.148203:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949100.148207:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949100.148227:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11374 health :500 00000800:00000200:0.0:1784949100.148481:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949100.148495:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:0.0:1784949100.148508:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949100.148524:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949100.148536:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949100.148543:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949100.148557:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949100.148578:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949100.148583:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949100.148591:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949100.148603:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949100.148623:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949100.148635:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949100.148650:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949100.148657:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3628:3628:241:4294967295] 192.168.202.137@tcp LPNI seq info [2326:2326:8:4294967295] 00000400:00000200:2.0:1784949100.148670:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949100.148677:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949100.148684:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000313a63a1. 00000800:00000200:2.0:1784949100.148692:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949100.148700:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949100.148705:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000313a63a1 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949100.148766:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949100.148845:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949100.148850:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000313a63a1. 00000400:00000200:2.0:1784949100.148856:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949100.148863:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949100.148880:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949100.148881:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949100.148900:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949100.148935:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949100.148940:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949100.148945:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949100.148959:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8e81 00000800:00000001:2.0:1784949100.148966:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949100.148975:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949100.148979:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949100.148982:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949100.148988:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000400:00000010:2.0:1784949100.148992:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000400:00000200:2.0:1784949100.148996:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949100.149000:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000177dc44a. 00000400:00000010:2.0:1784949100.149004:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 00000800:00000001:2.0:1784949100.149011:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949100.149014:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949100.788059:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949100.788069:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949100.788077:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949100.788079:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949100.788206:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949100.788215:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949100.788221:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949100.980151:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949100.980176:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949100.980180:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949100.980189:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949100.980194:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949100.980202:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949100.980205:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949101.812041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949101.812051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949101.812060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949101.812102:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949101.812106:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949101.812149:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949101.812154:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949101.812156:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949101.812159:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949101.812162:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949101.812304:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949101.812311:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949101.812315:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784949102.004151:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949102.004165:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949102.004176:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009c295fd6. 00000020:00000040:1.0:1784949102.004207:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949102.004235:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949102.004242:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949102.004280:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949102.004296:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000600b3b47. 00000100:00000001:1.0:1784949102.004328:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949102.004360:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949102.004379:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949102.004383:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949102.004402:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c295fd6] to pc [ptlrpcd_00_01+1] req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.004437:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949102.004440:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949102.004445:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949102.004448:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949102.004452:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949102.004456:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001725b510. 00000020:00000040:1.0:1784949102.004459:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949102.004462:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949102.004464:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949102.004467:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949102.004473:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000be1f3156. 00000100:00000001:1.0:1784949102.004477:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949102.004488:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949102.004498:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949102.004500:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949102.004502:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949102.004505:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001725b510] to pc [ptlrpcd_00_02+2] req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949102.004516:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949102.004524:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949102.004538:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.004545:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949102.004557:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11379) 00000100:00000001:0.0:1784949102.004571:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949102.004577:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949102.004581:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949102.004587:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1784949102.004589:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:2.0:1784949102.004592:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949102.004595:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949102.004597:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949102.004600:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949102.004613:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949102.004614:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949102.004620:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.004628:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949102.004629:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000009c295fd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289674112:0@lo:400: 00000100:00000001:1.0:1784949102.004633:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949102.004638:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949102.004649:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949102.004649:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949102.004664:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949102.004665:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949102.004670:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949102.004673:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949102.004675:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949102.004678:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949102.004681:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000001725b510 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289674240:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949102.004686:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949102.004693:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949102.004694:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949102.004698:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949102.004706:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1784949102.004707:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949102.004712:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 02000000:00000001:1.0:1784949102.004718:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949102.004721:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006620b0a1. 00000400:00000010:1.0:1784949102.004727:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000789ee428. 02000000:00000001:0.0:1784949102.004731:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949102.004740:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000010:0.0:1784949102.004745:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 00000100:00000200:1.0:1784949102.004753:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289674240, portal 10 00000100:00000001:1.0:1784949102.004759:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:0.0:1784949102.004761:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000001:1.0:1784949102.004762:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637813632 : -115544071737984 : ffff96e9cca5ed80) 00000100:00000200:0.0:1784949102.004779:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289674112, portal 25 00000100:00000040:1.0:1784949102.004780:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949102.004793:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949102.004803:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949102.004804:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637814528 : -115544071737088 : ffff96e9cca5f100) 00000100:00000040:1.0:1784949102.004811:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949102.004821:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 00000100:00000200:1.0:1784949102.004827:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289674240, offset 0 00000100:00000040:0.0:1784949102.004829:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949102.004840:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949102.004859:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949102.004865:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949102.004872:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000200:0.0:1784949102.004879:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289674112, offset 0 00000400:00000200:0.0:1784949102.004889:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949102.004897:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949102.004937:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949102.004944:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949102.004950:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949102.004965:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949102.004979:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3629:3629:241:4294967295] 192.168.202.137@tcp LPNI seq info [2327:2327:8:4294967295] 00000400:00000200:1.0:1784949102.004994:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949102.005003:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949102.005010:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005c010cf6. 00000800:00000200:1.0:1784949102.005025:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949102.005028:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:1.0:1784949102.005041:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949102.005049:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005c010cf6 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949102.005077:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949102.005269:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949102.005277:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005c010cf6. 00000400:00000200:0.0:1784949102.005792:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b180 00000400:00000200:0.0:1784949102.005809:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8481 [2] + 0 00000400:00000010:0.0:1784949102.005819:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006be72eea. 00000400:00000200:0.0:1784949102.005829:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bc3ccb92 00000400:00000200:2.0:1784949102.005837:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784949102.005845:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2172 usecs (3 jiffies) 00000400:00000200:0.0:1784949102.005846:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949102.005858:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949102.005862:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306041348 00000400:00000200:2.0:1784949102.005869:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0c9b918 00000400:00000200:0.0:1784949102.005876:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000010:2.0:1784949102.005877:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f0c9b918. 00000100:00000001:3.0:1784949102.005877:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784949102.005879:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:3.0:1784949102.005898:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949102.005902:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784949102.005905:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc3ccb92 00000100:00000001:1.0:1784949102.005949:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949102.005951:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000400:00000010:0.0:1784949102.005966:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bc3ccb92. 00000100:00000001:1.0:1784949102.005971:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306041349 00000100:00000001:3.0:1784949102.005972:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:2.0:1784949102.005973:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949102.005976:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949102.005988:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949102.005990:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000010:1.0:1784949102.005993:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:3.0:1784949102.006010:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949102.006013:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000e164874b x1871651289674112 msgsize 224 00000100:00000001:1.0:1784949102.006014:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949102.006021:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949102.006038:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949102.006039:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949102.006041:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306041349 00000100:00000040:2.0:1784949102.006063:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949102.006064:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949102.006065:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000010:1.0:1784949102.006065:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000040:0.0:1784949102.006097:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949102.006099:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949102.006102:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949102.006137:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306041349 00000100:00000001:2.0:1784949102.006142:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:1.0:1784949102.006146:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784949102.006150:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949102.006153:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949102.006164:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949102.006164:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949102.006169:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000100:00000001:1.0:1784949102.006179:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949102.006180:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000800:00000001:2.0:1784949102.006187:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.006195:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949102.006197:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949102.006198:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949102.006205:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949102.006207:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:2.0:1784949102.006212:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949102.006214:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949102.006216:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289674112 02000000:00000001:3.0:1784949102.006229:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949102.006240:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784949102.006242:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949102.006249:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949102.006258:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949102.006267:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949102.006273:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949102.006288:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b200 00000100:00100000:3.0:1784949102.006293:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289674112 00000100:00000001:0.0:1784949102.006295:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949102.006312:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784949102.006313:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1784949102.006314:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 169488 00000020:00000040:3.0:1784949102.006328:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949102.006335:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949102.006337:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000001:2.0:1784949102.006340:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000040:3.0:1784949102.006350:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949102.006353:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306041388 00000800:00000001:2.0:1784949102.006359:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949102.006364:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:2.0:1784949102.006371:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784949102.006374:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949102.006376:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306041348 00000400:00000200:2.0:1784949102.006393:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949102.006403:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306041388 00000100:00000001:3.0:1784949102.006403:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949102.006405:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949102.006422:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949102.006425:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306041349 00000100:00000010:2.0:1784949102.006433:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a6e2dac1. 00000020:00000010:3.0:1784949102.006435:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784949102.006437:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306041388 00000100:00000040:2.0:1784949102.006460:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000a6e2dac1 x1871651289674240 msgsize 224 00000100:00000001:3.0:1784949102.006460:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949102.006462:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306041349 00000020:00000010:3.0:1784949102.006491:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00000001:0.0:1784949102.006495:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306041388 00000100:00100000:2.0:1784949102.006514:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949102.006517:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306041349 00000020:00000010:3.0:1784949102.006520:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000089ec28da. 00000100:00000001:0.0:1784949102.006529:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949102.006532:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000096bcfddb. 00000100:00000001:0.0:1784949102.006538:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949102.006541:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949102.006547:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949102.006552:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949102.006553:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949102.006560:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000001:2.0:1784949102.006562:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949102.006568:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949102.006574:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949102.006576:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1784949102.006578:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949102.006579:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949102.006599:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949102.006622:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949102.006631:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949102.006634:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949102.006639:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949102.006654:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949102.006661:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949102.006668:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289674240 02000000:00000001:1.0:1784949102.006673:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949102.006674:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.006678:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949102.006683:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949102.006691:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949102.006698:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289674240 00000100:00000001:3.0:1784949102.006698:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949102.006709:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000001:1.0:1784949102.006710:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949102.006719:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949102.006731:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:3.0:1784949102.006733:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 776 00000020:00000040:1.0:1784949102.006743:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:3.0:1784949102.006747:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000020:00000001:1.0:1784949102.006751:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949102.006753:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782158928 : -115538927392688 : ffff96eaff466850) 00000020:00000001:1.0:1784949102.006771:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:3.0:1784949102.006779:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e164874b x1871651289674112/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:8/0 lens 224/0 e 0 to 0 dl 1784949113 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.006781:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949102.006791:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000d6df185. 00000100:00000001:1.0:1784949102.006804:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949102.006805:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949102.006820:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000d6df185. 00000020:00000010:3.0:1784949102.006821:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:3.0:1784949102.006843:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949102.006850:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006841dfc4. 00000020:00000010:3.0:1784949102.006851:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000020:00000010:1.0:1784949102.006864:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000169a7180. 00000100:00100000:3.0:1784949102.006866:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e164874b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289674112:12345-0@lo:400: 00000020:00000010:1.0:1784949102.006884:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000200:3.0:1784949102.006886:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289674112 00000020:00000001:3.0:1784949102.006900:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949102.006957:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:1.0:1784949102.006960:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:3.0:1784949102.006963:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949102.006971:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000001:3.0:1784949102.006973:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949102.006976:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:3.0:1784949102.006978:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949102.006987:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949102.006990:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949102.006994:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949102.006998:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949102.007003:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949102.007006:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949102.007014:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949102.007014:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949102.007026:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949102.007034:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000010:3.0:1784949102.007036:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000980961c7. 00000100:00000001:1.0:1784949102.007039:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949102.007046:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949102.007046:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949102.007062:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949102.007067:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949102.007071:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289674112 00000100:00000001:1.0:1784949102.007074:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949102.007077:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949102.007088:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e164874b x1871651289674112/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:8/0 lens 224/224 e 0 to 0 dl 1784949113 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.007094:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949102.007100:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000001:3.0:1784949102.007137:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949102.007147:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949102.007151:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 752 00000100:00001000:3.0:1784949102.007156:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=124 v=5 (1 1 1 1) 00000100:00000040:1.0:1784949102.007160:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949102.007168:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562018560 : -115544147533056 : ffff96e9c8216300) 00000100:00000001:3.0:1784949102.007168:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949102.007185:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:1.0:1784949102.007197:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a6e2dac1 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:8/0 lens 224/0 e 0 to 0 dl 1784949113 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949102.007197:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949102.007209:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949102.007214:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949102.007231:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949102.007246:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949102.007258:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000200:3.0:1784949102.007272:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289674112, offset 224 00000400:00000200:3.0:1784949102.007297:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949102.007313:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949102.007322:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:1.0:1784949102.007323:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000d6df185. 00000100:00000001:1.0:1784949102.007334:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949102.007345:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b180 00000020:00000010:1.0:1784949102.007346:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000d6df185. 00000400:00000200:3.0:1784949102.007361:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8e89 [1] + 224 00000100:00100000:1.0:1784949102.007377:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a6e2dac1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289674240:12345-192.168.202.137@tcp:400: 00000400:00000200:3.0:1784949102.007384:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949102.007399:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:1.0:1784949102.007401:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289674240 00000100:00000001:3.0:1784949102.007409:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1784949102.007413:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000200:3.0:1784949102.007417:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949102.007426:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949102.007438:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949102.007440:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949102.007454:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949102.007468:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949102.007479:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949102.007481:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1784949102.007485:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949102.007490:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949102.007494:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949102.007498:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949102.007499:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000020:00000001:1.0:1784949102.007503:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000010:3.0:1784949102.007505:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:3.0:1784949102.007517:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949102.007519:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949102.007525:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 02000000:00000001:1.0:1784949102.007526:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949102.007540:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000095d8d756. 00000100:00000001:3.0:1784949102.007544:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949102.007549:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949102.007558:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949102.007560:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949102.007583:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949102.007585:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949102.007596:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949102.007598:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949102.007600:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289674240 00000100:00000001:0.0:1784949102.007610:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949102.007614:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00010000:00000001:1.0:1784949102.007616:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000001:0.0:1784949102.007625:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1784949102.007634:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e164874b x1871651289674112/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:8/0 lens 224/224 e 0 to 0 dl 1784949113 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784949102.007637:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a6e2dac1 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:8/0 lens 224/224 e 0 to 0 dl 1784949113 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949102.007639:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949102.007650:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000400:00000200:0.0:1784949102.007667:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:0.0:1784949102.007680:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:0.0:1784949102.007698:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949102.007721:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949102.007762:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00100000:3.0:1784949102.007772:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e164874b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289674112:12345-0@lo:400: Request processed in 956us (1744us total) trans 0 rc 0/0 00010000:00000001:1.0:1784949102.007775:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949102.007793:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=119 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949102.007803:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.007820:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949102.007836:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949102.007847:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:3.0:1784949102.007858:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 776 02000000:00000001:1.0:1784949102.007867:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949102.007878:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949102.007880:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949102.007898:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949102.007900:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949102.007900:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949102.007944:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949102.007950:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949102.007953:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000096bcfddb. 00000100:00000001:0.0:1784949102.007960:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:1.0:1784949102.007970:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784949102.007975:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:3.0:1784949102.007976:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000400:00000010:1.0:1784949102.007979:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000100:00000001:0.0:1784949102.007989:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949102.007998:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949102.007998:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000089ec28da. 00000100:00000200:1.0:1784949102.008000:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289674240, offset 224 02000000:00000001:0.0:1784949102.008016:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949102.008020:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:0.0:1784949102.008027:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=124 v=5 (5 5 5 5) 00000400:00000200:1.0:1784949102.008034:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949102.008040:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784949102.008051:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000980961c7. 00000100:00001000:0.0:1784949102.008053:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=124 v=5 (1 1 1 1) 00000400:00000200:1.0:1784949102.008064:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:3.0:1784949102.008068:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949102.008072:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949102.008080:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949102.008082:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949102.008096:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3630:3630:241:4294967295] 192.168.202.137@tcp LPNI seq info [2328:2328:8:4294967295] 00010000:00000001:0.0:1784949102.008103:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:3.0:1784949102.008124:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:3.0:1784949102.008133:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e8bdea01. 00010000:00000001:0.0:1784949102.008144:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949102.008149:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000059548475. 00000100:00000001:0.0:1784949102.008156:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949102.008162:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:0.0:1784949102.008176:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949102.008185:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949102.008196:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e706e181. 00000100:00000001:0.0:1784949102.008214:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1784949102.008216:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949102.008226:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949102.008231:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949102.008234:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:0.0:1784949102.008241:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949102.008243:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e706e181 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:0.0:1784949102.008264:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000009c295fd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289674112:0@lo:400: 00000100:00000001:1.0:1784949102.008270:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949102.008273:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949102.008279:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949102.008281:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000040:0.0:1784949102.008287:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000009c295fd6 x1871651289674112/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949102.008289:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949102.008294:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949102.008302:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949102.008314:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:1.0:1784949102.008315:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a6e2dac1 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:8/0 lens 224/224 e 0 to 0 dl 1784949113 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949102.008321:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949102.008328:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006620b0a1. 02000000:00000001:0.0:1784949102.008342:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1784949102.008349:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a6e2dac1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289674240:12345-192.168.202.137@tcp:400: Request processed in 1029us (1882us total) trans 0 rc 0/0 00000020:00000001:0.0:1784949102.008350:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949102.008356:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949102.008365:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949102.008370:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000600b3b47. 00000100:00100000:1.0:1784949102.008377:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 752 02000000:00000001:0.0:1784949102.008380:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1784949102.008384:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 02000000:00000001:0.0:1784949102.008385:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949102.008392:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000009c295fd6. 00000100:00000001:1.0:1784949102.008393:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949102.008400:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949102.008403:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949102.008407:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949102.008411:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000169a7180. 00000100:00000001:0.0:1784949102.008412:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949102.008424:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949102.008426:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949102.008433:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949102.008435:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006841dfc4. 00000100:00000001:0.0:1784949102.008437:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949102.008445:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949102.008446:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949102.008452:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949102.008483:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949102.008493:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e706e181. 00000400:00000200:2.0:1784949102.008500:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949102.008511:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949102.008521:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:2.0:1784949102.008528:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000800:00000001:3.0:1784949102.008528:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949102.008538:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949102.008541:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949102.008552:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949102.008559:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949102.008568:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949102.008581:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949102.008597:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b200 00000400:00000200:3.0:1784949102.008605:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8e85 [1] + 224 00000800:00000001:3.0:1784949102.008612:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949102.008627:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949102.008630:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949102.008635:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949102.008641:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949102.008647:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949102.008660:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949102.008693:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949102.008702:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949102.008705:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.008748:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949102.008753:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949102.008760:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949102.008763:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949102.008766:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000789ee428. 00000400:00000200:1.0:1784949102.008772:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784949102.008774:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:1.0:1784949102.008778:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949102.008786:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949102.008798:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.008808:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949102.008810:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.008812:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949102.008815:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949102.008817:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949102.008819:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949102.008821:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949102.008827:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=119 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949102.008831:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=119 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949102.008834:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949102.008836:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949102.008838:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949102.008842:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949102.008851:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949102.008855:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949102.008857:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949102.008859:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949102.008861:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949102.008865:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.008874:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949102.008876:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949102.008878:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949102.008881:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949102.008889:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000001725b510 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289674240:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949102.008894:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.008896:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949102.008898:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000001725b510 x1871651289674240/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949102.008940:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949102.008943:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949102.008945:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 02000000:00000001:1.0:1784949102.008949:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949102.008951:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949102.008952:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949102.008955:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949102.008957:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000be1f3156. 02000000:00000001:1.0:1784949102.008960:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949102.008962:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949102.008963:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000001725b510. 00000100:00000001:1.0:1784949102.008967:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949102.008968:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.008971:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949102.008976:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949102.008977:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949102.008980:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949102.008982:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949102.049037:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306041388, at 4306041389 00000100:00000010:2.0:1784949102.049057:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:1.0:1784949102.049067:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306041388, at 4306041389 00000100:00000040:0.0:1784949102.049076:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306041388, at 4306041389 00000100:00000010:1.0:1784949102.049086:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:0.0:1784949102.049093:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:3.0:1784949102.049152:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306041388, at 4306041389 00000100:00000010:3.0:1784949102.049176:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784949102.836037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949102.836047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949102.836056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949102.836057:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949102.836175:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949102.836184:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949102.836191:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949103.028154:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949103.028175:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949103.028179:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949103.028188:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949103.028193:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949103.028198:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949103.028202:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949103.860041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949103.860051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949103.860060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949103.860061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949103.860207:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949103.860213:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949103.860218:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949104.052148:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949104.052165:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949104.052168:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949104.052173:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949104.052177:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949104.052181:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949104.052184:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949104.884028:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949104.884038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949104.884047:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949104.884048:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949104.884173:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949104.884180:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949104.884184:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949105.076039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949105.076057:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949105.076059:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949105.076066:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949105.076069:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949105.076073:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949105.076077:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949105.907976:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949105.907983:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949105.907990:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949105.907991:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949105.908069:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949105.908093:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949105.908098:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949106.100029:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949106.100038:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949106.100057:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949081/real 1784949081] req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949106 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949106.100100:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949106.100104:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949106.100125:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000930107ef. 00000400:00000200:0.0:1784949106.100134:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:0.0:1784949106.100139:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000100:00000001:0.0:1784949106.100148:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949106.100157:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949106 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949106.100169:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949106 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949106.100187:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949106.100190:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.100194:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949106.100197:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949106.100202:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949106 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949106.100212:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.100223:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949106.100225:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.100230:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949106.100234:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949106.100238:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949106.100244:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949106 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949106.100254:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949106.100258:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.100260:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949106.100262:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.100266:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949106.100269:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949106.100273:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949106.100301:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:0.0:1784949106.100307:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949106.100315:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949106 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949106.100326:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006a448376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289673088:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784949106.100335:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.100341:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949106.100345:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006a448376 x1871651289673088/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949106 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949106.100355:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949106.100363:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1784949106.100363:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:0.0:1784949106.100370:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d78419ed. 00000100:00000001:3.0:1784949106.100381:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:0.0:1784949106.100385:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000010:3.0:1784949106.100392:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000020:00000001:0.0:1784949106.100393:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949106.100404:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000040:3.0:1784949106.100405:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949106.100419:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1784949106.100421:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:3.0:1784949106.100425:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:0.0:1784949106.100427:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000cc641d8a. 02000000:00000001:3.0:1784949106.100436:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949106.100437:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949106.100446:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949106.100451:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006a448376. 02000000:00000010:3.0:1784949106.100453:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:0.0:1784949106.100463:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949106.100468:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949106.100468:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.100480:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.100486:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784949106.100488:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949106.100496:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.100509:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949106.100531:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949106.100532:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949106.100538:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949106.100548:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_00_03+3] req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949106.100612:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.100619:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784949106.100621:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:1.0:1784949106.100627:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949106.100633:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1784949106.100636:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:1.0:1784949106.100643:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949106.100644:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784949106.100653:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784949106.100664:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1784949106.100680:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949106.100681:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949106.100689:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949106.100689:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11353 00000100:00000001:0.0:1784949106.100692:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949106.100708:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949106.100712:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949106.100717:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949106.100722:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00100000:1.0:1784949106.100728:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289674368:0@lo:400: 00000100:00080000:3.0:1784949106.100733:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11352 00000100:00000001:1.0:1784949106.100741:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949106.100752:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949106.100754:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949106.100757:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949106.100762:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 02000000:00000001:1.0:1784949106.100766:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949106.100770:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000400:00000010:1.0:1784949106.100772:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000789ee428. 00000400:00000200:3.0:1784949106.100780:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000400:00000010:1.0:1784949106.100786:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000040:3.0:1784949106.100794:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949106.100801:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1784949106.100802:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289674368, portal 25 00000100:00000001:1.0:1784949106.100817:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1784949106.100818:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000040:3.0:1784949106.100822:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949106.100824:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000100:00000001:3.0:1784949106.100840:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:0.0:1784949106.100842:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000040:1.0:1784949106.100844:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949106.100854:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:0.0:1784949106.100861:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006620b0a1. 00000020:00000040:3.0:1784949106.100863:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:1.0:1784949106.100866:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1784949106.100869:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784949106.100874:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000040:1.0:1784949106.100878:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000040:3.0:1784949106.100880:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:1.0:1784949106.100883:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000020:00000001:3.0:1784949106.100887:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:0.0:1784949106.100890:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000012554f3a. 00000100:00000200:1.0:1784949106.100890:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289674368, offset 0 00000020:00000001:3.0:1784949106.100897:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:0.0:1784949106.100960:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000040:3.0:1784949106.100962:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949106.100969:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784949106.100977:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:3.0:1784949106.100982:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:3.0:1784949106.100987:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:3.0:1784949106.100992:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:3.0:1784949106.101002:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000200:0.0:1784949106.101004:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000400:00000200:1.0:1784949106.101005:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00080000:3.0:1784949106.101025:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000200:0.0:1784949106.101031:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784949106.101033:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949106.101042:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1784949106.101048:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:3.0:1784949106.101054:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949106.101062:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1784949106.101065:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949106.101089:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:3.0:1784949106.101094:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1784949106.101102:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949106.101103:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3631:3631:241:4294967295] 1.2.3.4@tcp LPNI seq info [109:109:8:4294967295] 00000100:00000010:3.0:1784949106.101115:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003492edb3. 00000020:00000040:3.0:1784949106.101123:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:0.0:1784949106.101124:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 02000000:00000001:3.0:1784949106.101130:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784949106.101134:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949106.101139:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949106.101144:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949106.101149:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949106.101151:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 02000000:00000010:3.0:1784949106.101159:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000980961c7. 00000100:00000001:3.0:1784949106.101171:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949106.101172:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000400:00000200:1.0:1784949106.101172:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b280 00000800:00000010:0.0:1784949106.101190:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000662152ce. 00000100:00100000:3.0:1784949106.101194:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784949106.101199:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x849d [2] + 0 00000800:00000200:0.0:1784949106.101219:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:1.0:1784949106.101230:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008946a238. 00000400:00000200:1.0:1784949106.101245:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000001d10ae9a 00000800:00000200:0.1:1784949106.101255:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:1.0:1784949106.101263:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949106.101274:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949106.101278:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949106.101282:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949106.101284:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:3.0:1784949106.101297:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003492edb3] to pc [ptlrpcd_rcv+-1] req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949106.101298:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001d10ae9a 00000400:00000200:0.0:1784949106.101304:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 00000400:00000010:1.0:1784949106.101306:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000001d10ae9a. 00000100:00000001:1.0:1784949106.101321:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949106.101329:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784949106.101342:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000000ae6b462 x1871651289674368 msgsize 224 00000100:00100000:1.0:1784949106.101354:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949106.101356:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949106.101365:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101366:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:3.0:1784949106.101374:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1784949106.101376:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:1.0:1784949106.101383:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000100:00000001:3.0:1784949106.101388:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949106.101391:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949106.101392:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1784949106.101394:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000074c559e1. 00000100:00000001:2.0:1784949106.101398:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:1.0:1784949106.101401:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949106.101405:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949106.101406:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949106.101422:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949106.101422:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949106.101429:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949106.101433:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784949106.101436:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949106.101455:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000040:1.0:1784949106.101465:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949106.101469:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949106.101502:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949106.101522:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949106.101525:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949106.101536:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949106.101539:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289674368 02000000:00000001:0.0:1784949106.101551:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:2.0:1784949106.101555:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949106.101564:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949106.101566:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949106.101568:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949106.101580:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101583:0:337122:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784949106.101592:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101602:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949106.101606:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949106.101615:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289674496:1.2.3.4@tcp:8: 00000100:00000001:1.0:1784949106.101623:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306045483 00000100:00000001:3.0:1784949106.101625:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949106.101639:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.101641:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1784949106.101643:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000040:3.0:1784949106.101669:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000074c559e1] to pc [ptlrpcd_00_00+0] req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.101670:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101676:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949106.101680:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101687:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306045483 00000100:00100000:0.0:1784949106.101695:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289674368 00000100:00000001:2.0:1784949106.101697:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949106.101704:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000020:00000001:0.0:1784949106.101706:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949106.101709:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949106.101719:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949106.101720:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949106.101723:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784949106.101734:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949106.101734:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306045483 00000100:00000001:2.0:1784949106.101735:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949106.101745:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:1.0:1784949106.101747:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:1.0:1784949106.101757:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000001:0.0:1784949106.101760:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949106.101760:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949106.101762:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306045483 00000100:00000001:2.0:1784949106.101763:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1784949106.101769:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:2.0:1784949106.101769:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949106.101770:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949106.101774:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.101777:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101778:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949106.101782:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949106.101796:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949106.101797:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11383) 00000100:00000001:1.0:1784949106.101807:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101814:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101816:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.101820:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949106.101822:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949106.101825:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949106.101825:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949106.101828:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949106.101830:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949106.101834:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784949106.101837:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949106.101838:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.101843:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.101852:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949106.101853:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949106.101856:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949106.101857:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949106.101859:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949106.101863:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949106.101864:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289674624:192.168.202.137@tcp:400: 00000020:00000010:0.0:1784949106.101865:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:3.0:1784949106.101870:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.101875:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949106.101878:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949106.101880:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949106.101883:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949106.101886:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 02000000:00000001:2.0:1784949106.101886:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949106.101893:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000008fae13bf. 02000000:00000001:2.0:1784949106.101896:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949106.101899:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000400:00000010:2.0:1784949106.101902:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000919ac10c. 00000020:00000010:0.0:1784949106.101908:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f26470de. 00000020:00000010:0.0:1784949106.101916:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000010:2.0:1784949106.101958:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000100:00000200:2.0:1784949106.101966:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289674624, portal 10 00000100:00000001:2.0:1784949106.101969:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949106.101971:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554956160 : -115544154595456 : ffff96e9c7b59f80) 00000100:00000040:2.0:1784949106.101979:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.101988:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949106.101993:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949106.102013:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000100:00000200:2.0:1784949106.102018:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289674624, offset 0 00000400:00000200:2.0:1784949106.102025:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949106.102041:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:0.0:1784949106.102049:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:2.0:1784949106.102050:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949106.102059:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949106.102064:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:2.0:1784949106.102065:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949106.102117:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784949106.102118:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:2.0:1784949106.102122:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3632:3632:240:4294967295] 192.168.202.137@tcp LPNI seq info [2329:2329:8:4294967295] 00000100:00000001:0.0:1784949106.102124:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949106.102133:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949106.102137:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949106.102138:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949106.102145:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000313a63a1. 00000800:00000200:2.0:1784949106.102150:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949106.102156:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784949106.102162:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949106.102162:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000313a63a1 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949106.102179:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949106.102186:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2176 usecs (3 jiffies) 00000100:00000001:2.0:1784949106.102189:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306045445 00000100:00000001:2.0:1784949106.102192:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306045483 00000100:00000001:2.0:1784949106.102195:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306045445 00000100:00000001:2.0:1784949106.102198:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306045483 00000100:00000001:2.0:1784949106.102201:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306045445 00000100:00000001:2.0:1784949106.102204:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306045483 00000100:00000001:2.0:1784949106.102205:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306045445 00000100:00000001:2.0:1784949106.102208:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306045483 00000100:00000001:2.0:1784949106.102210:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949106.102211:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949106.102213:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.102217:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949106.102219:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.102224:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.102233:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1784949106.102234:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 777 00000100:00000001:3.0:1784949106.102237:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949106.102238:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:0.0:1784949106.102242:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949106.102244:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949106.102250:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.102252:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782160976 : -115538927390640 : ffff96eaff467050) 00000100:00000001:2.0:1784949106.102254:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949106.102258:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.102261:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949106.102268:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949106.102269:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000ae6b462 x1871651289674368/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:12/0 lens 224/0 e 0 to 0 dl 1784949117 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.102273:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.102290:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949106.102295:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:0.0:1784949106.102300:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949106.102305:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000800:00000200:1.0:1784949106.102308:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949106.102313:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000313a63a1. 00000100:00100000:0.0:1784949106.102318:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000ae6b462 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289674368:12345-0@lo:400: 00000400:00000200:1.0:1784949106.102321:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1784949106.102329:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289674368 00000800:00000001:3.0:1784949106.102333:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949106.102334:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784949106.102337:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:1.0:1784949106.102362:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000020:00000001:0.0:1784949106.102364:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:1.0:1784949106.102367:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000800:00000001:3.0:1784949106.102371:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949106.102372:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949106.102375:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:3.0:1784949106.102392:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949106.102394:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000200:1.0:1784949106.102398:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949106.102402:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949106.102404:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949106.102423:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949106.102437:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949106.102440:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:0.0:1784949106.102461:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784949106.102465:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784949106.102470:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:1.0:1784949106.102473:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949106.102481:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949106.102488:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b380 00000020:00000001:0.0:1784949106.102505:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949106.102517:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:3.0:1784949106.102526:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 169712 00000100:00000001:1.0:1784949106.102529:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949106.102533:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949106.102547:0:337113:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:0.0:1784949106.102551:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009925ca9f. 00000800:00000001:3.0:1784949106.102554:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:0.0:1784949106.102564:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949106.102569:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.102571:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949106.102577:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949106.102582:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949106.102593:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000040:0.0:1784949106.102596:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289674368 00000100:00000001:3.0:1784949106.102605:0:337114:0:(events.c:286:request_in_callback()) Process entered 00010000:00000001:0.0:1784949106.102608:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000200:3.0:1784949106.102610:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:3.0:1784949106.102621:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001445c4ff. 00010000:00000200:0.0:1784949106.102632:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000ae6b462 x1871651289674368/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:12/0 lens 224/224 e 0 to 0 dl 1784949117 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949106.102633:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000001445c4ff x1871651289674624 msgsize 224 00000100:00100000:3.0:1784949106.102658:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:0.0:1784949106.102672:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949106.102675:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949106.102680:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=128 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949106.102682:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949106.102692:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:3.0:1784949106.102700:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949106.102701:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:3.0:1784949106.102706:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.102709:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949106.102717:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949106.102720:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.102726:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949106.102730:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949106.102735:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000200:0.0:1784949106.102740:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289674368, offset 224 00000400:00000200:0.0:1784949106.102747:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784949106.102759:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949106.102763:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949106.102767:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289674624 02000000:00000001:2.0:1784949106.102770:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:0.0:1784949106.102772:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b280 00000100:00000001:2.0:1784949106.102772:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949106.102776:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949106.102781:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949106.102784:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289674624 00000400:00000200:0.0:1784949106.102787:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8ea1 [1] + 224 00000020:00000001:2.0:1784949106.102788:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949106.102791:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949106.102794:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949106.102797:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949106.102801:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784949106.102802:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949106.102803:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949106.102808:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949106.102812:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:2.0:1784949106.102813:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000001:2.0:1784949106.102818:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949106.102820:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00000001:0.0:1784949106.102821:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:2.0:1784949106.102826:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006b8bfa56. 00000100:00000200:0.0:1784949106.102830:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949106.102832:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000b5ba1cb. 00000020:00000010:2.0:1784949106.102837:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000040:2.0:1784949106.102845:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949106.102849:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949106.102850:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:0.0:1784949106.102853:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.102854:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.102860:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949106.102864:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949106.102866:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.102886:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949106.102887:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949106.102896:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784949106.102897:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000100:00000001:2.0:1784949106.102899:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:0.0:1784949106.102934:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00100000:2.0:1784949106.102959:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 753 00000100:00000001:0.0:1784949106.102963:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:2.0:1784949106.102967:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949106.102971:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949106.102972:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554962432 : -115544154589184 : ffff96e9c7b5b800) 00000100:00000001:0.0:1784949106.102984:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949106.102984:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001445c4ff x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:12/0 lens 224/0 e 0 to 0 dl 1784949117 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949106.103017:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949106.103017:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949106.103022:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949106.103026:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:2.0:1784949106.103030:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00010000:00000001:0.0:1784949106.103033:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:1.0:1784949106.103038:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784949106.103039:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949106.103041:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949106.103045:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:2.0:1784949106.103049:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000400:00000010:1.0:1784949106.103054:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000789ee428. 00000020:00000001:0.0:1784949106.103056:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00100000:2.0:1784949106.103065:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001445c4ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289674624:12345-192.168.202.137@tcp:400: 00000400:00000200:1.0:1784949106.103069:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000100:00000040:0.0:1784949106.103102:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000ae6b462 x1871651289674368/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:12/0 lens 224/224 e 0 to 0 dl 1784949117 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949106.103106:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289674624 00000400:00000010:1.0:1784949106.103108:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000020:00000001:2.0:1784949106.103113:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949106.103136:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1784949106.103137:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949106.103141:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949106.103146:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949106.103146:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949106.103150:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00100000:0.0:1784949106.103152:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000ae6b462 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289674368:12345-0@lo:400: Request processed in 857us (1804us total) trans 0 rc 0/0 00000020:00000001:2.0:1784949106.103156:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949106.103160:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949106.103165:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949106.103168:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949106.103173:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949106.103180:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 777 00000020:00000001:2.0:1784949106.103180:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000200:1.0:1784949106.103183:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.103186:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:0.0:1784949106.103190:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 02000000:00000001:2.0:1784949106.103191:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949106.103196:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007617a2e3. 00000100:00000001:0.0:1784949106.103199:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949106.103200:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1784949106.103200:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949106.103204:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:1.0:1784949106.103205:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.103206:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949106.103211:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949106.103213:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f26470de. 00000100:00000001:1.0:1784949106.103213:0:337122:0:(client.c:1580:after_reply()) Process entered 00010000:00000040:2.0:1784949106.103214:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289674624 02000000:00000001:1.0:1784949106.103218:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00010000:00000001:2.0:1784949106.103219:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949106.103222:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:0.0:1784949106.103224:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949106.103225:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:2.0:1784949106.103226:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001445c4ff x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:12/0 lens 224/224 e 0 to 0 dl 1784949117 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949106.103231:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949106.103234:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000100:00001000:1.0:1784949106.103240:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=128 v=5 (5 5 5 5) 00010000:00000001:2.0:1784949106.103243:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00001000:1.0:1784949106.103245:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=128 v=5 (1 1 1 1) 00010000:00000001:2.0:1784949106.103247:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949106.103248:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949106.103251:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00001000:2.0:1784949106.103252:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=123 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949106.103255:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949106.103257:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784949106.103258:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00010000:00000001:1.0:1784949106.103259:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000010:0.0:1784949106.103264:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009925ca9f. 00010000:00000001:1.0:1784949106.103264:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949106.103265:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949106.103270:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.103271:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949106.103273:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:2.0:1784949106.103275:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784949106.103278:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949106.103278:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949106.103279:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.103286:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:0.0:1784949106.103290:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:2.0:1784949106.103294:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949106.103296:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949106.103299:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:0.0:1784949106.103301:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a5d8678. 00000400:00000010:2.0:1784949106.103301:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000001:1.0:1784949106.103302:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949106.103310:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289674624, offset 224 00000100:00000040:1.0:1784949106.103313:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949106.103315:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bc3ccb92. 00000400:00000200:2.0:1784949106.103323:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784949106.103350:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289674368:0@lo:400: 00000400:00000200:2.0:1784949106.103358:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949106.103368:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949106.103371:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3633:3633:240:4294967295] 192.168.202.137@tcp LPNI seq info [2330:2330:8:4294967295] 00000100:00000001:1.0:1784949106.103378:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949106.103389:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289674368/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949106.103397:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949106.103419:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949106.103427:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005c3de731. 00000100:00000001:1.0:1784949106.103433:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000200:2.0:1784949106.103441:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:1.0:1784949106.103443:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949106.103450:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 00000800:00000200:2.0:1784949106.103454:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949106.103462:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005c3de731 type 1, nob 320 niov 1 nkiov 1 02000000:00000001:1.0:1784949106.103464:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949106.103469:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949106.103472:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949106.103477:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949106.103480:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 00000100:00000001:2.0:1784949106.103482:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949106.103488:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1784949106.103489:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949106.103496:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00010000:00000001:2.0:1784949106.103496:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000010:1.0:1784949106.103505:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000020:00000001:2.0:1784949106.103505:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949106.103512:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949106.103521:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001445c4ff x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:12/0 lens 224/224 e 0 to 0 dl 1784949117 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949106.103533:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00100000:2.0:1784949106.103534:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001445c4ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289674624:12345-192.168.202.137@tcp:400: Request processed in 511us (899us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949106.103541:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949106.103546:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949106.103553:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 753 00000100:00000001:1.0:1784949106.103559:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1784949106.103560:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949106.103565:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.103568:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949106.103571:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949106.103573:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:1.0:1784949106.103577:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949106.103579:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000b5ba1cb. 00000020:00000010:2.0:1784949106.103587:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949106.103592:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006b8bfa56. 00000020:00000040:2.0:1784949106.103630:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949106.103636:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949106.103771:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949106.103779:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005c3de731. 00000800:00000001:1.0:1784949106.103780:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949106.103788:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949106.103793:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949106.103799:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949106.103803:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949106.103805:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949106.103811:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:3.0:1784949106.103818:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000400:00000200:1.0:1784949106.103823:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949106.103830:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949106.103834:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949106.103838:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b380 00000400:00000200:1.0:1784949106.103848:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8ead [1] + 224 00000800:00000001:1.0:1784949106.103853:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949106.103863:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949106.103866:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949106.103870:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949106.103873:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949106.103876:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949106.103883:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949106.103903:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949106.103957:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949106.103958:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:1.0:1784949106.103964:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.103964:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949106.103973:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949106.103976:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949106.103980:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000400:00000200:2.0:1784949106.103986:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:2.0:1784949106.104005:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000100:00000001:2.0:1784949106.104010:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949106.104016:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949106.104027:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.104035:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949106.104037:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.104039:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949106.104042:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949106.104043:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949106.104045:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949106.104047:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949106.104052:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=123 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949106.104056:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=123 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949106.104058:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949106.104060:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949106.104062:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949106.104066:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949106.104112:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.104115:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949106.104116:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949106.104118:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949106.104119:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949106.104121:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.104126:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949106.104127:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949106.104128:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949106.104130:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949106.104135:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289674624:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949106.104138:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.104140:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949106.104142:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000074c559e1 x1871651289674624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949122 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949106.104146:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949106.104148:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949106.104149:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000008fae13bf. 02000000:00000001:2.0:1784949106.104153:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949106.104154:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949106.104155:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949106.104156:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949106.104157:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:2.0:1784949106.104160:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949106.104161:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949106.104162:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000074c559e1. 00000100:00000001:2.0:1784949106.104164:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949106.104165:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.104166:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949106.104169:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949106.104170:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949106.104172:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949106.104173:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949106.143998:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306045483, at 4306045484 00000100:00000040:2.0:1784949106.144000:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306045483, at 4306045484 00000100:00000010:1.0:1784949106.144014:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:2.0:1784949106.144017:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:3.0:1784949106.144050:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306045483, at 4306045484 00000100:00000040:0.0:1784949106.144052:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306045483, at 4306045484 00000100:00000010:3.0:1784949106.144094:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:0.0:1784949106.144100:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000800:00000200:2.0:1784949106.292114:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:2.0:1784949106.292127:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000800:00000200:2.0:1784949106.292134:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000800:00000200:3.0:1784949106.292321:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784949106.932046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949106.932056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949106.932064:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949106.932091:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949106.932174:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949106.932180:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949106.932185:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949107.124044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949107.124055:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949107.124105:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949106/real 0] req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949107 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949107.124131:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.124137:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949107.124142:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949107.124152:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949107 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949107.124167:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.124181:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949107.124183:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.124187:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949107.124192:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949107.124196:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949107.124204:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949107 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949107.124219:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949107.124226:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.124229:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949107.124231:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.124236:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949107.124240:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949107.124246:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949107.124285:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:2.0:1784949107.124291:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949107.124298:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949107 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949107.124315:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289674496:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949107.124324:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.124331:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949107.124336:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003492edb3 x1871651289674496/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949107 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949107.124351:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1784949107.124358:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949107.124361:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784949107.124367:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949107.124375:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000980961c7. 00000100:00000040:3.0:1784949107.124379:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1784949107.124386:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949107.124395:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1784949107.124400:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949107.124402:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003492edb3. 00000100:00000001:2.0:1784949107.124417:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000010:3.0:1784949107.124417:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006e540bde. 00000100:00000001:2.0:1784949107.124431:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784949107.124433:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949107.124448:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949107.124450:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949107.124457:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949107.124465:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784949107.124469:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.124473:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.124481:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949107.124483:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949107.124489:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000001:3.0:1784949107.124507:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949107.124524:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949107.124547:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949107.124552:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949107.124563:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e540bde] to pc [ptlrpcd_00_01+1] req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949107.124624:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949107.124628:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949107.124633:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784949107.124636:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784949107.124638:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784949107.124643:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784949107.124652:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11353 00000400:00000200:3.0:1784949107.124667:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784949107.124670:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784949107.124676:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11378 00000100:00100000:2.0:1784949107.124680:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1784949107.124685:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949107.124688:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:2.0:1784949107.124694:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1784949107.124695:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1784949107.124703:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949107.124709:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949107.124711:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949107.124714:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949107.124727:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.124732:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949107.124732:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949107.124747:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949107.124748:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949107.124757:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000040:2.0:1784949107.124762:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949107.124768:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949107.124776:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:3.0:1784949107.124790:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949107.124801:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:3.0:1784949107.124817:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:3.0:1784949107.124842:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:2.0:1784949107.124843:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949107.124851:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949107.124855:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000001:2.0:1784949107.124862:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949107.124863:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949107.124871:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949107.124873:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949107.124890:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:3.0:1784949107.124895:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:2.0:1784949107.124898:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949107.124953:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289674752:0@lo:400: 00000100:00000001:2.0:1784949107.124960:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949107.124970:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949107.124973:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949107.124977:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1784949107.124978:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00080000:3.0:1784949107.124986:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 02000000:00000010:2.0:1784949107.124987:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006b8bfa56. 00000100:00000001:3.0:1784949107.124999:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949107.125000:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949107.125023:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1784949107.125026:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000919ac10c. 02000000:00000001:3.0:1784949107.125036:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949107.125039:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 02000000:00000001:3.0:1784949107.125050:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949107.125056:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:2.0:1784949107.125057:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289674752, portal 25 00000100:00000001:3.0:1784949107.125061:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.125093:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949107.125099:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554957056 : -115544154594560 : ffff96e9c7b5a300) 00000100:00000010:3.0:1784949107.125102:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000020:00000040:3.0:1784949107.125109:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:2.0:1784949107.125112:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949107.125120:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784949107.125130:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.125150:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949107.125150:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949107.125160:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1784949107.125169:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784949107.125171:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949107.125177:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 02000000:00000010:3.0:1784949107.125183:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d02b883a. 00000100:00000200:2.0:1784949107.125185:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289674752, offset 0 00000100:00000001:3.0:1784949107.125195:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949107.125211:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949107.125217:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949107.125248:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949107.125256:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949107.125265:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949107.125267:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006d868603] to pc [ptlrpcd_rcv+-1] req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949107.125302:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b400 00000100:00000001:3.0:1784949107.125310:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949107.125323:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x84bd [2] + 0 00000100:00000040:3.0:1784949107.125323:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949107.125358:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000010:2.0:1784949107.125359:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b743d1b9. 00000100:00000010:3.0:1784949107.125366:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000400:00000200:2.0:1784949107.125369:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bed828fe 00000020:00000040:3.0:1784949107.125375:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949107.125380:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949107.125382:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949107.125384:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784949107.125384:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949107.125399:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949107.125402:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949107.125412:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949107.125419:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:3.0:1784949107.125421:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000400:00000200:2.0:1784949107.125422:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bed828fe 00000400:00000010:2.0:1784949107.125440:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bed828fe. 00000100:00000001:3.0:1784949107.125443:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949107.125444:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949107.125451:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949107.125457:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949107.125461:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784949107.125479:0:337119:0:(events.c:337:request_in_callback()) incoming req@000000003b52fa03 x1871651289674752 msgsize 224 00000100:00100000:2.0:1784949107.125512:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949107.125521:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949107.125550:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949107.125554:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949107.125558:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949107.125566:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1784949107.125569:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 02000000:00000001:0.0:1784949107.125570:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949107.125577:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949107.125579:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 02000000:00000001:0.0:1784949107.125582:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949107.125595:0:337119:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1784949107.125597:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949107.125605:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_02+2] req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949107.125615:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949107.125627:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289674880:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784949107.125661:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949107.125668:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:0.0:1784949107.125676:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1784949107.125681:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289674752 02000000:00000001:0.0:1784949107.125691:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949107.125696:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949107.125708:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949107.125712:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949107.125720:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949107.125732:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009925ca9f. 02000000:00000001:1.0:1784949107.125737:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.125738:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949107.125742:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949107.125749:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289674752 00000100:00000040:2.0:1784949107.125750:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949107.125754:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000681ff0f0. 00000020:00000001:1.0:1784949107.125757:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949107.125761:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949107.125764:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000010:0.0:1784949107.125774:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000020:00000001:1.0:1784949107.125777:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:3.0:1784949107.125785:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11384) 00000020:00000040:1.0:1784949107.125786:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000200:0.0:1784949107.125791:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289674880, portal 4 00000020:00000001:1.0:1784949107.125807:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949107.125808:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949107.125810:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949107.125821:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949107.125822:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554951680 : -115544154599936 : ffff96e9c7b58e00) 00000100:00000001:3.0:1784949107.125823:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949107.125825:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949107.125832:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949107.125834:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949107.125847:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949107.125850:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949107.125852:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306046507 00000100:00000040:0.0:1784949107.125861:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949107.125869:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784949107.125870:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f59314ac. 00000020:00000010:1.0:1784949107.125871:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:1.0:1784949107.125897:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949107.125897:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949107.125975:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306046508 00000020:00000010:1.0:1784949107.125988:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00000010:2.0:1784949107.126000:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008cd34e64. 00000020:00000010:1.0:1784949107.126016:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004520da55. 00000100:00000001:2.0:1784949107.126020:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:1.0:1784949107.126039:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000169a7180. 00000100:00000001:2.0:1784949107.126041:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306046508 00000100:00000001:3.0:1784949107.126043:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949107.126057:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949107.126058:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949107.126061:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 00000020:00000010:1.0:1784949107.126063:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:2.0:1784949107.126102:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000001:3.0:1784949107.126106:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949107.126111:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949107.126117:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949107.126119:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949107.126119:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306046508 00000400:00000010:0.0:1784949107.126123:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000100:00000001:1.0:1784949107.126143:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:3.0:1784949107.126145:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949107.126147:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b9b6859c. 00000100:00000200:0.0:1784949107.126149:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289674880, offset 0 00000100:00000001:1.0:1784949107.126152:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949107.126163:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.126168:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00100000:3.0:1784949107.126170:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289675008:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949107.126179:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949107.126189:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949107.126190:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949107.126203:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949107.126209:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949107.126212:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949107.126214:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949107.126221:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949107.126223:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949107.126225:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000001:3.0:1784949107.126235:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949107.126239:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949107.126241:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949107.126248:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000010:3.0:1784949107.126252:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000c405a384. 00000400:00000200:0.0:1784949107.126259:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1784949107.126264:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949107.126266:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 778 00000100:00000040:1.0:1784949107.126282:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:0.0:1784949107.126283:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000010:3.0:1784949107.126284:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000de368614. 00000100:00000001:1.0:1784949107.126290:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782157904 : -115538927393712 : ffff96eaff466450) 00000100:00000001:2.0:1784949107.126300:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949107.126306:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949107.126306:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000400:00000200:0.0:1784949107.126309:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1784949107.126310:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003b52fa03 x1871651289674752/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:13/0 lens 224/0 e 0 to 0 dl 1784949118 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949107.126326:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675008, portal 10 00000400:00000200:0.0:1784949107.126333:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949107.126342:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949107.126353:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000100:00000001:1.0:1784949107.126359:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949107.126369:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3634:3634:240:4294967295] 192.168.202.137@tcp LPNI seq info [2331:2331:8:4294967295] 00000020:00000010:1.0:1784949107.126369:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000040:3.0:1784949107.126380:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949107.126382:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949107.126404:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00100000:1.0:1784949107.126439:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003b52fa03 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289674752:12345-0@lo:400: 00000100:00000001:3.0:1784949107.126452:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949107.126461:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000200:1.0:1784949107.126473:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289674752 00000100:00000040:3.0:1784949107.126484:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949107.126487:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949107.126498:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:3.0:1784949107.126502:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000800:00000200:0.0:1784949107.126505:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949107.126505:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949107.126519:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000200:3.0:1784949107.126521:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289675008, offset 0 00000020:00000001:1.0:1784949107.126525:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000010:0.0:1784949107.126527:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000635133da. 00000020:00000001:1.0:1784949107.126543:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949107.126548:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949107.126552:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949107.126561:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000200:0.0:1784949107.126568:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:1.0:1784949107.126571:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949107.126571:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949107.126592:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949107.126598:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000200:0.0:1784949107.126604:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 02000000:00000001:1.0:1784949107.126605:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:3.0:1784949107.126614:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000010:1.0:1784949107.126616:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008905542c. 00000800:00000200:0.0:1784949107.126623:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000635133da type 1, nob 616 niov 1 nkiov 1 02000000:00000001:1.0:1784949107.126633:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949107.126642:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949107.126646:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949107.126651:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949107.126653:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00010000:00000040:1.0:1784949107.126663:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289674752 00000400:00000200:3.0:1784949107.126665:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949107.126676:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949107.126688:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:3.0:1784949107.126690:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:0.0:1784949107.126693:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:0.0:1784949107.126712:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306046508 00010000:00000200:1.0:1784949107.126716:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003b52fa03 x1871651289674752/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:13/0 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949107.126720:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3635:3635:239:4294967295] 192.168.202.137@tcp LPNI seq info [2332:2332:7:4294967295] 00000100:00000001:0.0:1784949107.126738:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4306046507 00010000:00000001:1.0:1784949107.126772:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949107.126773:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306046508 00010000:00000001:1.0:1784949107.126783:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949107.126798:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306046508 00000400:00000200:3.0:1784949107.126802:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00001000:1.0:1784949107.126803:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=129 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949107.126816:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306046508 00000100:00000001:0.0:1784949107.126835:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306046508 00000100:00000001:1.0:1784949107.126838:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1784949107.126848:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949107.126854:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306046508 00000100:00000040:1.0:1784949107.126859:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000010:3.0:1784949107.126861:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000100:00000001:0.0:1784949107.126869:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306046508 00000100:00000001:1.0:1784949107.126880:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:0.0:1784949107.126882:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1784949107.126892:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:1.0:1784949107.126902:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949107.126956:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949107.126956:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949107.126967:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949107.126969:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000635133da. 00000100:00000001:1.0:1784949107.126971:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949107.126981:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949107.126989:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1784949107.126991:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:1.0:1784949107.126996:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949107.126999:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949107.127019:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000400:00000010:1.0:1784949107.127021:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000200:2.0:1784949107.127021:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949107.127023:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949107.127038:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1784949107.127049:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784949107.127051:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289674752, offset 224 00000400:00000200:2.0:1784949107.127053:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000100:00000001:0.0:1784949107.127054:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949107.127087:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949107.127091:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949107.127097:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000100:00000001:0.0:1784949107.127098:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949107.127102:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2177 usecs (3 jiffies) 00000100:00000001:3.0:1784949107.127117:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306046470 00000100:00000001:2.0:1784949107.127122:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949107.127123:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306046508 00000400:00000200:1.0:1784949107.127137:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949107.127140:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306046470 00000100:00000001:3.0:1784949107.127152:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306046508 00000100:00000200:2.0:1784949107.127154:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949107.127156:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949107.127168:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306046470 00000400:00000200:1.0:1784949107.127178:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:0.0:1784949107.127180:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949107.127183:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306046508 00000100:00000001:3.0:1784949107.127197:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306046470 00000800:00000001:0.0:1784949107.127200:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949107.127210:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306046508 00000800:00000001:0.0:1784949107.127216:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949107.127220:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b400 00000100:00000001:3.0:1784949107.127229:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949107.127250:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949107.127251:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8ec1 [1] + 224 00000400:00000200:0.0:1784949107.127258:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949107.127271:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.127284:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949107.127286:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949107.127287:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949107.127311:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643b480 00000100:00000040:2.0:1784949107.127311:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949132 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949107.127313:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949107.127315:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949107.127323:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949107.127338:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949107.127344:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000100:0.0:1784949107.127351:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289674880 offset 0 length 520: 4 00000100:00000001:3.0:1784949107.127353:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:1.0:1784949107.127354:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949107.127367:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949107.127371:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000ee641b2a is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:3.0:1784949107.127388:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949107.127389:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:2.0:1784949107.127421:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.127428:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784949107.127430:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949107.127432:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949107.127448:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949107.127457:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949107.127463:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784949107.127475:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784949107.127478:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:1.0:1784949107.127488:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949107.127492:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949107.127504:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949107.127505:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.127514:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784949107.127516:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784949107.127524:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949107.127527:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784949107.127529:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1784949107.127532:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:2.0:1784949107.127537:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1784949107.127543:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003b52fa03 x1871651289674752/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:13/0 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949107.127548:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000400:00000200:2.0:1784949107.127563:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000100:00100000:1.0:1784949107.127566:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003b52fa03 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289674752:12345-0@lo:400: Request processed in 1205us (2084us total) trans 0 rc 0/0 00000400:00000010:2.0:1784949107.127572:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000100:00100000:1.0:1784949107.127583:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 778 00000100:00000001:2.0:1784949107.127583:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784949107.127591:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949107.127596:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:2.0:1784949107.127599:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949107.127600:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949107.127609:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000169a7180. 00000020:00000010:1.0:1784949107.127618:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949107.127628:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004520da55. 00000020:00000040:1.0:1784949107.127638:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:2.0:1784949107.127639:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949107.127649:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949107.127658:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008905542c. 02000000:00000001:1.0:1784949107.127673:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949107.127673:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949107.127682:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.127685:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949107.127692:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784949107.127694:0:337119:0:(client.c:1580:after_reply()) Process entered 00000400:00000010:1.0:1784949107.127706:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000789ee428. 00000800:00000200:3.0:1784949107.127706:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000001:2.0:1784949107.127707:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000800:00000001:0.0:1784949107.127710:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949107.127727:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000010:3.0:1784949107.127729:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000010:1.0:1784949107.127732:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004dd24a2e. 00000800:00000001:0.0:1784949107.127734:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.127743:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949107.127746:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949107.127748:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784949107.127754:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949107.127759:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949107.127770:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00001000:2.0:1784949107.127772:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=129 v=5 (5 5 5 5) 00000400:00000200:3.0:1784949107.127790:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000400:00000200:0.0:1784949107.127791:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00001000:2.0:1784949107.127795:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=129 v=5 (1 1 1 1) 00000800:00000001:1.0:1784949107.127797:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949107.127801:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000100:00000001:3.0:1784949107.127822:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949107.127823:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949107.127832:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949107.127841:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949107.127847:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b500 00010000:00000001:2.0:1784949107.127851:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949107.127862:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949107.127884:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949107.127902:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 169936 00000100:00000001:3.0:1784949107.127947:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949107.127960:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949107.127971:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949107.127987:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949107.128015:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949107.128018:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949107.128032:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949107.128033:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949107.128050:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949107.128054:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949107.128089:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949107.128093:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784949107.128096:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949107.128099:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949107.128112:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000047c147a0. 00000100:00000040:2.0:1784949107.128113:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949107.128121:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000047c147a0 x1871651289675008 msgsize 224 00000100:00100000:0.0:1784949107.128146:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00100000:2.0:1784949107.128161:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289674752:0@lo:400: 00000100:00000001:0.0:1784949107.128173:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949107.128175:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.128183:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:0.0:1784949107.128187:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949107.128190:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006e540bde x1871651289674752/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949107.128198:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949107.128215:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949107.128221:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949107.128229:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006b8bfa56. 02000000:00000001:2.0:1784949107.128244:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1784949107.128247:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:2.0:1784949107.128249:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00100000:3.0:1784949107.128253:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675008 00000020:00000040:2.0:1784949107.128257:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 02000000:00000001:3.0:1784949107.128260:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949107.128270:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:2.0:1784949107.128274:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784949107.128276:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949107.128286:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:3.0:1784949107.128287:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949107.128305:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675008 02000000:00000001:2.0:1784949107.128306:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1784949107.128310:0:343544:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1784949107.128317:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:3.0:1784949107.128318:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000010:2.0:1784949107.128333:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006e540bde. 00000020:00000001:3.0:1784949107.128333:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949107.128341:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784949107.128351:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1784949107.128351:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949107.128362:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949107.128362:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949107.128373:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949107.128376:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949107.128382:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:3.0:1784949107.128387:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949107.128391:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:3.0:1784949107.128391:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00000001:2.0:1784949107.128405:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949107.128413:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009802f36f. 00000100:00000001:2.0:1784949107.128416:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:3.0:1784949107.128423:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dae37bd3. 00000100:00000001:2.0:1784949107.128426:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949107.128432:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000040:3.0:1784949107.128446:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949107.128451:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949107.128453:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949107.128457:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949107.128469:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949107.128475:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949107.128478:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949107.128503:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949107.128524:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949107.128528:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949107.128539:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 754 00000100:00000040:3.0:1784949107.128546:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949107.128551:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554976128 : -115544154575488 : ffff96e9c7b5ed80) 00000100:00000040:3.0:1784949107.128560:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000047c147a0 x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:13/0 lens 224/0 e 0 to 0 dl 1784949118 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949107.128579:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949107.128585:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:3.0:1784949107.128590:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949107.128593:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00100000:3.0:1784949107.128600:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000047c147a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289675008:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949107.128606:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675008 00000020:00000001:3.0:1784949107.128611:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949107.128616:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949107.128621:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949107.128624:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949107.128628:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949107.128633:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949107.128637:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949107.128641:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949107.128644:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949107.128647:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949107.128650:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949107.128656:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949107.128660:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949107.128664:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b10320f2. 02000000:00000001:3.0:1784949107.128667:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949107.128670:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949107.128673:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949107.128676:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675008 00010000:00000001:3.0:1784949107.128680:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949107.128686:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000047c147a0 x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:13/0 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949107.128696:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949107.128700:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949107.128705:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=124 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949107.128710:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949107.128713:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949107.128717:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949107.128720:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949107.128722:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949107.128726:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949107.128729:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949107.128732:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000100:00000200:3.0:1784949107.128737:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289675008, offset 224 00000400:00000200:3.0:1784949107.128743:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949107.128754:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949107.128761:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3636:3636:240:4294967295] 192.168.202.137@tcp LPNI seq info [2333:2333:8:4294967295] 00000400:00000200:3.0:1784949107.128774:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949107.128780:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949107.128784:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784949107.128790:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949107.128796:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949107.128801:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949107.128821:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949107.128826:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949107.128829:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949107.128831:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949107.128834:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949107.128838:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000047c147a0 x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:13/0 lens 224/224 e 0 to 0 dl 1784949118 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949107.128851:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000047c147a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289675008:12345-192.168.202.137@tcp:400: Request processed in 271us (727us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949107.128866:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 754 00000100:00000040:3.0:1784949107.128872:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949107.128876:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949107.128881:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949107.128888:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dae37bd3. 00000020:00000010:3.0:1784949107.128893:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949107.128897:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009802f36f. 00000020:00000040:3.0:1784949107.128942:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949107.128948:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949107.129024:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949107.129032:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:1.0:1784949107.129040:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949107.129051:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949107.129061:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000800:00000001:3.0:1784949107.129089:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784949107.129090:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000100:00000001:1.0:1784949107.129102:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949107.129104:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949107.129113:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949107.129119:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949107.129123:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949107.129137:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949107.129152:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b500 00000400:00000200:3.0:1784949107.129160:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8ed1 [1] + 224 00000800:00000001:3.0:1784949107.129169:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949107.129175:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949107.129179:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949107.129183:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949107.129190:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949107.129195:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949107.129208:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949107.129234:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949107.129244:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949107.129248:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949107.129301:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949107.129308:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949107.129316:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949107.129319:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949107.129322:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000de368614. 00000400:00000200:0.0:1784949107.129332:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:0.0:1784949107.129335:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:0.0:1784949107.129339:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949107.129348:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949107.129362:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949107.129373:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949107.129375:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949107.129377:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949107.129380:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949107.129382:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949107.129385:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949107.129387:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949107.129395:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=124 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949107.129400:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=124 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949107.129403:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949107.129404:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949107.129406:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949107.129411:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949107.129422:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949107.129426:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949107.129428:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949107.129431:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949107.129432:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949107.129436:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949107.129444:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949107.129446:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949107.129448:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949107.129451:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949107.129459:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289675008:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949107.129465:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949107.129467:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949107.129470:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289675008/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949107.129478:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949107.129480:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949107.129483:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000c405a384. 02000000:00000001:0.0:1784949107.129489:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949107.129491:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949107.129492:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949107.129495:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949107.129497:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 02000000:00000001:0.0:1784949107.129500:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949107.129502:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949107.129504:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:0.0:1784949107.129508:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949107.129510:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949107.129512:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949107.129518:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949107.129519:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949107.129522:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949107.129525:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949107.169033:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306046508, at 4306046509 00000100:00000010:1.0:1784949107.169054:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00000100:00000040:3.0:1784949107.169095:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306046508, at 4306046509 00000100:00000040:2.0:1784949107.169105:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306046508, at 4306046509 00000100:00000040:0.0:1784949107.169106:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306046508, at 4306046509 00000100:00000010:3.0:1784949107.169117:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b9b6859c. 00000100:00000010:2.0:1784949107.169125:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000010:0.0:1784949107.169131:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f59314ac. 00010000:00000001:2.0:1784949107.956105:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949107.956115:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949107.956124:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949107.956126:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949107.956202:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949107.956208:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949107.956213:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949108.980044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949108.980101:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949108.980110:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949108.980112:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949108.980193:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949108.980200:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949108.980204:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949110.004021:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949110.004031:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949110.004062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949110.004064:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949110.004137:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949110.004145:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949110.004151:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949111.028029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949111.028074:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949111.028084:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949111.028089:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949111.028092:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949111.028096:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949111.028099:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949111.028102:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949111.028105:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949111.028107:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949111.028227:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949111.028235:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949111.028239:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949112.052051:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949112.052062:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949112.052070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949112.052072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949112.052194:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949112.052201:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949112.052205:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949112.244061:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949112.244076:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949112.244079:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949112.244086:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949112.244090:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949112.244094:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949112.244098:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949112.244139:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949112.244150:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949112.244159:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fcdc5039. 00000020:00000040:2.0:1784949112.244167:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949112.244174:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949112.244178:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949112.244185:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949112.244195:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fd30a9ad. 00000100:00000001:2.0:1784949112.244204:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949112.244216:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949112.244231:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949112.244234:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949112.244240:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcdc5039] to pc [ptlrpcd_00_03+3] req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949112.244280:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949112.244284:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949112.244289:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949112.244294:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949112.244298:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949112.244302:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004ef1e045. 00000020:00000040:2.0:1784949112.244306:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949112.244309:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949112.244310:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949112.244313:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.244317:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1784949112.244318:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fbc83221. 00000100:00000001:1.0:1784949112.244326:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949112.244327:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.244334:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949112.244338:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949112.244342:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949112.244346:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949112.244348:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949112.244384:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949112.244389:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.244394:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1784949112.244401:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ef1e045] to pc [ptlrpcd_00_00+0] req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949112.244402:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949112.244414:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949112.244420:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949112.244426:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949112.244431:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fcdc5039 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289675136:0@lo:400: 00000100:00000001:1.0:1784949112.244438:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949112.244446:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949112.244447:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949112.244450:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949112.244457:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784949112.244458:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11389) 02000000:00000010:1.0:1784949112.244466:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 02000000:00000001:1.0:1784949112.244472:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949112.244480:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba749c18. 00000400:00000010:1.0:1784949112.244490:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000001:0.0:1784949112.244500:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784949112.244501:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675136, portal 25 00000100:00000001:3.0:1784949112.244502:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949112.244508:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1784949112.244511:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949112.244513:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949112.244515:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949112.244516:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562010496 : -115544147541120 : ffff96e9c8214380) 00000100:00000001:0.0:1784949112.244530:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949112.244531:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949112.244537:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949112.244548:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949112.244554:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949112.244555:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949112.244558:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949112.244565:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949112.244570:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000100:00000040:0.0:1784949112.244573:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949112.244576:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289675136, offset 0 00000400:00000200:1.0:1784949112.244597:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949112.244602:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949112.244606:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949112.244610:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949112.244615:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949112.244624:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:0.0:1784949112.244625:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949112.244637:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004ef1e045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289675264:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949112.244643:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:1.0:1784949112.244649:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b580 02000000:00000001:0.0:1784949112.244652:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949112.244658:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949112.244662:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949112.244668:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x84d9 [2] + 0 02000000:00000010:0.0:1784949112.244672:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 00000400:00000010:1.0:1784949112.244676:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006785ee40. 02000000:00000001:0.0:1784949112.244679:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949112.244685:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000efb93606 00000400:00000010:0.0:1784949112.244688:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ee18875c. 00000400:00000010:0.0:1784949112.244698:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000400:00000200:1.0:1784949112.244700:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1784949112.244703:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675264, portal 10 00000100:00000001:0.0:1784949112.244710:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949112.244712:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562019456 : -115544147532160 : ffff96e9c8216680) 00000100:00000040:0.0:1784949112.244724:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949112.244728:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949112.244741:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000efb93606 00000100:00000001:0.0:1784949112.244746:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784949112.244748:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000efb93606. 00000100:00000040:0.0:1784949112.244756:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949112.244759:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000400:00000010:0.0:1784949112.244761:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00000200:1.0:1784949112.244765:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:0.0:1784949112.244768:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289675264, offset 0 00000100:00000040:1.0:1784949112.244773:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000e3680464 x1871651289675136 msgsize 224 00000400:00000200:0.0:1784949112.244782:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784949112.244794:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949112.244800:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784949112.244805:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949112.244812:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949112.244819:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949112.244824:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949112.244829:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949112.244830:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949112.244838:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3637:3637:240:4294967295] 192.168.202.137@tcp LPNI seq info [2334:2334:8:4294967295] 00000400:00000200:0.0:1784949112.244855:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949112.244864:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949112.244871:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000057c7ab4. 00000400:00000200:1.0:1784949112.244875:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:1.0:1784949112.244885:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000800:00000200:0.0:1784949112.244886:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949112.244889:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949112.244900:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675136 00000100:00000001:1.0:1784949112.244902:0:337122:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1784949112.244947:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000200:0.0:1784949112.244948:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784949112.244955:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:0.0:1784949112.244960:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000057c7ab4 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949112.244963:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949112.244971:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949112.244976:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949112.244989:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675136 00000100:00000001:0.0:1784949112.244993:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949112.244996:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949112.245003:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:0.0:1784949112.245008:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2180 usecs (3 jiffies) 00000020:00000001:2.0:1784949112.245011:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949112.245019:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949112.245041:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306051588 00000100:00000010:0.0:1784949112.245050:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000040:1.0:1784949112.245056:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949112.245064:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949112.245070:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.245075:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949112.245077:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306051588 00000100:00000010:0.0:1784949112.245084:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:1.0:1784949112.245086:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949112.245089:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949112.245090:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949112.245096:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306051627 00000020:00000001:2.0:1784949112.245102:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949112.245104:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306051588 00000020:00000001:2.0:1784949112.245111:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949112.245112:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306051588 00000100:00000010:0.0:1784949112.245113:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:2.0:1784949112.245128:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949112.245129:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949112.245131:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306051627 00000020:00000010:2.0:1784949112.245138:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:0.0:1784949112.245142:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306051588 00000100:00000001:1.0:1784949112.245143:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306051588 00000100:00000001:2.0:1784949112.245146:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949112.245155:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000100:00000010:0.0:1784949112.245158:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:1.0:1784949112.245160:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306051627 00000020:00000010:2.0:1784949112.245168:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006b8bfa56. 00000100:00000001:0.0:1784949112.245170:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949112.245172:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306051588 00000020:00000010:2.0:1784949112.245186:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008e9164dd. 00000100:00000001:0.0:1784949112.245188:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949112.245189:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306051627 00000020:00000010:2.0:1784949112.245198:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:0.0:1784949112.245201:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.245203:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306051588 00000100:00000001:0.0:1784949112.245207:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949112.245215:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.245216:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949112.245217:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949112.245227:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949112.245229:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949112.245235:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949112.245236:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949112.245240:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.245243:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949112.245252:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.245254:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949112.245261:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949112.245267:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949112.245270:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:0.0:1784949112.245273:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949112.245275:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949112.245276:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949112.245287:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949112.245294:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949112.245302:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949112.245305:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:0.0:1784949112.245310:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949112.245311:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:3.0:1784949112.245315:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00100000:2.0:1784949112.245330:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 779 00000800:00000010:3.0:1784949112.245330:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000057c7ab4. 00000400:00000200:0.0:1784949112.245332:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:2.0:1784949112.245343:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:3.0:1784949112.245346:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949112.245350:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782162000 : -115538927389616 : ffff96eaff467450) 00000400:00000200:0.0:1784949112.245360:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b600 00000400:00000200:3.0:1784949112.245366:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:2.0:1784949112.245368:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e3680464 x1871651289675136/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:18/0 lens 224/0 e 0 to 0 dl 1784949123 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949112.245386:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 170160 00000400:00000200:3.0:1784949112.245405:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000800:00000001:0.0:1784949112.245407:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784949112.245413:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:3.0:1784949112.245420:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949112.245429:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949112.245431:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949112.245439:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949112.245455:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949112.245460:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949112.245465:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784949112.245473:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949112.245475:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949112.245487:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bb1b4c23. 00000100:00000040:0.0:1784949112.245497:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000bb1b4c23 x1871651289675264 msgsize 224 00000100:00000001:3.0:1784949112.245509:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949112.245517:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1784949112.245519:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784949112.245520:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949112.245532:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:2.0:1784949112.245538:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949112.245542:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000100:00100000:2.0:1784949112.245552:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e3680464 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289675136:12345-0@lo:400: 00000100:00000001:0.0:1784949112.245555:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:2.0:1784949112.245560:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675136 00000020:00000001:2.0:1784949112.245567:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:0.0:1784949112.245572:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949112.245573:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949112.245581:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949112.245583:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949112.245585:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949112.245593:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949112.245598:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:2.0:1784949112.245599:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949112.245605:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949112.245608:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675264 00000020:00000001:2.0:1784949112.245611:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000001:3.0:1784949112.245614:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:2.0:1784949112.245617:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949112.245620:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:2.0:1784949112.245622:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949112.245626:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949112.245628:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 02000000:00000001:3.0:1784949112.245636:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949112.245637:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949112.245642:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:3.0:1784949112.245645:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675264 02000000:00000010:2.0:1784949112.245650:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000032069f11. 00000020:00000001:3.0:1784949112.245651:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949112.245657:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000001:2.0:1784949112.245659:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949112.245664:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949112.245666:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949112.245672:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949112.245673:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949112.245681:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675136 00000020:00000001:3.0:1784949112.245683:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:2.0:1784949112.245688:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:3.0:1784949112.245691:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000200:2.0:1784949112.245698:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e3680464 x1871651289675136/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:18/0 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949112.245699:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949112.245708:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000100:00000001:3.0:1784949112.245716:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949112.245725:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00010000:00000001:2.0:1784949112.245727:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:3.0:1784949112.245733:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009802f36f. 00010000:00000001:2.0:1784949112.245735:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949112.245742:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b0a11306. 00000100:00001000:2.0:1784949112.245746:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=134 v=5 (1 1 1 1) 00000020:00000010:3.0:1784949112.245751:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:2.0:1784949112.245760:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949112.245768:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:2.0:1784949112.245770:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949112.245776:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949112.245781:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949112.245782:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949112.245789:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949112.245793:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784949112.245799:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000001:2.0:1784949112.245801:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949112.245806:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949112.245811:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949112.245812:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949112.245821:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949112.245830:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000100:00000001:3.0:1784949112.245836:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949112.245839:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289675136, offset 224 00000100:00000001:3.0:1784949112.245856:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784949112.245857:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949112.245864:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949112.245879:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 755 00000400:00000200:2.0:1784949112.245880:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949112.245889:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949112.245895:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554979712 : -115544154571904 : ffff96e9c7b5fb80) 00000100:00000040:3.0:1784949112.245939:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bb1b4c23 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:18/0 lens 224/0 e 0 to 0 dl 1784949123 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949112.245943:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b580 00000400:00000200:2.0:1784949112.245963:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8ee5 [1] + 224 00000100:00000001:3.0:1784949112.245967:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784949112.245974:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949112.245978:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000be8613d0. 00000400:00000200:2.0:1784949112.245984:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949112.245991:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949112.245994:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949112.246002:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000be8613d0. 00000100:00000200:2.0:1784949112.246006:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949112.246058:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bb1b4c23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289675264:12345-192.168.202.137@tcp:400: 00000100:00000040:2.0:1784949112.246065:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949112.246075:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675264 00000020:00000001:3.0:1784949112.246085:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949112.246093:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949112.246097:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949112.246099:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1784949112.246103:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949112.246111:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:2.0:1784949112.246112:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:2.0:1784949112.246118:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000020:00000001:3.0:1784949112.246123:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:2.0:1784949112.246126:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949112.246131:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:3.0:1784949112.246133:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949112.246141:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949112.246142:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949112.246151:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:2.0:1784949112.246152:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949112.246154:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949112.246160:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949112.246161:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949112.246162:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949112.246169:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949112.246172:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000001:2.0:1784949112.246174:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:1.0:1784949112.246177:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949112.246183:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949112.246184:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:2.0:1784949112.246187:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e3680464 x1871651289675136/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:18/0 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949112.246189:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ba749c18. 02000000:00000001:3.0:1784949112.246193:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:1.0:1784949112.246203:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 02000000:00000010:3.0:1784949112.246207:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f77b2f51. 00000400:00000010:1.0:1784949112.246211:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00100000:2.0:1784949112.246217:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e3680464 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289675136:12345-0@lo:400: Request processed in 691us (1436us total) trans 0 rc 0/0 02000000:00000001:3.0:1784949112.246221:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.246223:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949112.246235:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949112.246236:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949112.246243:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 779 00000020:00000001:3.0:1784949112.246252:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949112.246256:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00010000:00000040:3.0:1784949112.246267:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675264 00000100:00000001:2.0:1784949112.246268:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949112.246274:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:1.0:1784949112.246276:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949112.246280:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000010:2.0:1784949112.246285:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008e9164dd. 00000020:00000010:2.0:1784949112.246297:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00010000:00000200:3.0:1784949112.246298:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bb1b4c23 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:18/0 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949112.246308:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006b8bfa56. 00000100:00000001:1.0:1784949112.246313:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949112.246319:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784949112.246326:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949112.246329:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949112.246338:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:2.0:1784949112.246338:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949112.246344:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000010:2.0:1784949112.246346:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000032069f11. 00010000:00000001:3.0:1784949112.246348:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949112.246350:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949112.246360:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949112.246361:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949112.246361:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949112.246367:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784949112.246377:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=134 v=5 (5 5 5 5) 00000100:00000200:2.0:1784949112.246377:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:3.0:1784949112.246377:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=129 v=5 (1 1 1 1) 00000400:00000010:2.0:1784949112.246388:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000919ac10c. 00000100:00001000:1.0:1784949112.246398:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=134 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949112.246401:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1784949112.246402:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000037419771. 00000100:00000001:1.0:1784949112.246407:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000040:3.0:1784949112.246414:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949112.246415:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949112.246424:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784949112.246426:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00010000:00000001:1.0:1784949112.246428:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.246436:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949112.246439:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949112.246446:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949112.246447:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949112.246459:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949112.246465:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949112.246474:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1784949112.246475:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949112.246478:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949112.246486:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949112.246487:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000200:3.0:1784949112.246501:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289675264, offset 224 00000100:00100000:1.0:1784949112.246504:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fcdc5039 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289675136:0@lo:400: 00000100:00000001:1.0:1784949112.246515:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949112.246518:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784949112.246525:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949112.246526:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fcdc5039 x1871651289675136/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949112.246540:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949112.246544:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949112.246547:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 00000400:00000200:3.0:1784949112.246555:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:1.0:1784949112.246556:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949112.246561:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949112.246566:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949112.246573:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:3.0:1784949112.246575:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3638:3638:240:4294967295] 192.168.202.137@tcp LPNI seq info [2335:2335:8:4294967295] 02000000:00000010:1.0:1784949112.246577:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fd30a9ad. 02000000:00000001:1.0:1784949112.246587:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949112.246592:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949112.246594:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fcdc5039. 00000100:00000001:1.0:1784949112.246599:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949112.246603:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949112.246609:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949112.246618:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949112.246619:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949112.246625:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949112.246633:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949112.246639:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949112.246645:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949112.246656:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949112.246667:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949112.246677:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949112.246683:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949112.246713:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949112.246720:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949112.246726:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949112.246729:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949112.246733:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949112.246741:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bb1b4c23 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:18/0 lens 224/224 e 0 to 0 dl 1784949123 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949112.246761:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bb1b4c23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289675264:12345-192.168.202.137@tcp:400: Request processed in 794us (1261us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949112.246777:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 755 00000100:00000040:3.0:1784949112.246782:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949112.246786:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949112.246790:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949112.246796:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b0a11306. 00000020:00000010:3.0:1784949112.246804:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949112.246812:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009802f36f. 00000020:00000040:3.0:1784949112.246820:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949112.246826:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949112.246943:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949112.246950:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:0.0:1784949112.246959:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949112.246968:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949112.246974:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:0.0:1784949112.246977:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000800:00000001:3.0:1784949112.246981:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949112.246983:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949112.246989:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949112.247001:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949112.247010:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949112.247019:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949112.247064:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949112.247084:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b600 00000400:00000200:3.0:1784949112.247095:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8eed [1] + 224 00000800:00000001:3.0:1784949112.247105:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949112.247114:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949112.247119:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949112.247127:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949112.247135:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949112.247141:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949112.247161:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949112.247195:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949112.247207:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949112.247213:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949112.247244:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949112.247250:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949112.247256:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949112.247258:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949112.247260:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ee18875c. 00000400:00000200:0.0:1784949112.247266:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:0.0:1784949112.247268:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00000001:0.0:1784949112.247271:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949112.247278:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949112.247291:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949112.247301:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949112.247302:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949112.247305:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949112.247307:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949112.247309:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949112.247311:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949112.247314:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949112.247320:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=129 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949112.247324:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=129 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949112.247328:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949112.247330:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949112.247333:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949112.247336:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949112.247346:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949112.247349:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949112.247351:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949112.247353:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949112.247355:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949112.247358:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949112.247366:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949112.247369:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949112.247371:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949112.247374:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949112.247382:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004ef1e045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289675264:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949112.247387:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949112.247390:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949112.247392:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004ef1e045 x1871651289675264/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949112.247399:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949112.247401:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949112.247403:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 02000000:00000001:0.0:1784949112.247409:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949112.247411:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949112.247413:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949112.247415:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949112.247417:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fbc83221. 02000000:00000001:0.0:1784949112.247420:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949112.247422:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949112.247424:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004ef1e045. 00000100:00000001:0.0:1784949112.247429:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949112.247430:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949112.247432:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949112.247437:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949112.247439:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949112.247441:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949112.247444:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949112.287996:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306051627, at 4306051628 00000100:00000010:0.0:1784949112.288011:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:2.0:1784949112.288090:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306051627, at 4306051628 00000100:00000040:3.0:1784949112.288093:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306051627, at 4306051628 00000100:00000040:1.0:1784949112.288105:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306051627, at 4306051628 00000100:00000010:2.0:1784949112.288107:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:3.0:1784949112.288112:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000010:1.0:1784949112.288125:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00010000:00000001:2.0:1784949113.076097:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949113.076106:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949113.076114:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949113.076116:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949113.076193:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949113.076200:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949113.076205:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949113.268072:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949113.268087:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949113.268089:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949113.268094:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949113.268097:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949113.268101:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949113.268105:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949114.100090:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949114.100100:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949114.100108:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949114.100110:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949114.100197:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949114.100203:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949114.100208:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949114.292068:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949114.292085:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949114.292088:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949114.292095:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949114.292098:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949114.292104:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949114.292108:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949115.124055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949115.124063:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949115.124070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949115.124072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949115.124132:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949115.124137:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949115.124141:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949115.316053:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949115.316069:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949115.316072:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949115.316079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949115.316083:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949115.316089:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949115.316093:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949116.148064:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949116.148073:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949116.148080:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949116.148082:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949116.148219:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949116.148226:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949116.148231:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949116.340117:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949116.340135:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949116.340138:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949116.340146:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949116.340150:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949116.340154:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949116.340158:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949117.172077:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949117.172086:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949117.172094:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949117.172095:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949117.172187:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949117.172194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949117.172198:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949117.364028:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949117.364044:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949117.364047:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949117.364056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949117.364060:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949117.364065:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949117.364069:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949117.364196:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949117.364227:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949117.364236:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000020:00000040:3.0:1784949117.364245:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949117.364251:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949117.364256:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949117.364263:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949117.364273:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:3.0:1784949117.364283:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949117.364296:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949117.364311:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949117.364315:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949117.364324:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_00_01+1] req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949117.364369:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949117.364373:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:0.0:1784949117.364376:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784949117.364379:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1784949117.364386:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784949117.364388:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949117.364393:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949117.364396:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949117.364403:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000031b67b62. 00000100:00000040:0.0:1784949117.364406:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949117.364409:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949117.364419:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949117.364422:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949117.364425:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949117.364433:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949117.364434:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.364436:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949117.364443:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949117.364449:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 02000000:00000001:0.0:1784949117.364452:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949117.364456:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949117.364459:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949117.364465:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949117.364473:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289675392:0@lo:400: 00000100:00000001:0.0:1784949117.364480:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949117.364488:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949117.364491:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949117.364495:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949117.364501:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 02000000:00000001:0.0:1784949117.364504:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949117.364511:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ee18875c. 00000400:00000010:0.0:1784949117.364539:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000100:00000200:0.0:1784949117.364561:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675392, portal 25 00000100:00000001:0.0:1784949117.364564:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949117.364567:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554960640 : -115544154590976 : ffff96e9c7b5b100) 00000100:00000040:0.0:1784949117.364574:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949117.364585:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949117.364588:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949117.364592:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00000200:0.0:1784949117.364596:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289675392, offset 0 00000100:00000040:3.0:1784949117.364603:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784949117.364605:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949117.364621:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949117.364624:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949117.364631:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949117.364638:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000031b67b62] to pc [ptlrpcd_00_02+2] req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949117.364650:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b680 00000400:00000200:0.0:1784949117.364663:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x84f5 [2] + 0 00000400:00000010:0.0:1784949117.364669:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bb516699. 00000400:00000200:0.0:1784949117.364674:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000017500e72 00000100:00000001:3.0:1784949117.364679:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949117.364684:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784949117.364687:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11394) 00000400:00000200:0.0:1784949117.364695:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949117.364703:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000017500e72 00000400:00000010:0.0:1784949117.364706:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000017500e72. 00000100:00000001:0.0:1784949117.364713:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949117.364716:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784949117.364718:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949117.364720:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949117.364722:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000009bc35acb x1871651289675392 msgsize 224 00000100:00000001:1.0:1784949117.364725:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949117.364727:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.364732:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1784949117.364740:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949117.364746:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949117.364748:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949117.364763:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949117.364766:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.364768:0:337120:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:1.0:1784949117.364770:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949117.364776:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949117.364779:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949117.364780:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000100:00100000:1.0:1784949117.364786:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289675520:192.168.202.137@tcp:400: 00000400:00000010:0.0:1784949117.364787:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:0.0:1784949117.364798:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949117.364798:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949117.364807:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949117.364811:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949117.364812:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949117.364816:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949117.364824:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 02000000:00000001:1.0:1784949117.364830:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.364832:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:1.0:1784949117.364836:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba749c18. 00000100:00000040:0.0:1784949117.364838:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949117.364847:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000001:0.0:1784949117.364854:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949117.364856:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675520, portal 10 00000100:00000001:0.0:1784949117.364858:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949117.364863:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949117.364867:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554959744 : -115544154591872 : ffff96e9c7b5ad80) 00000100:00000001:0.0:1784949117.364871:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.364879:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306056746 00000100:00000040:1.0:1784949117.364880:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784949117.364890:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:1.0:1784949117.364895:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949117.364903:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949117.364961:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784949117.364966:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306056747 00000100:00000040:1.0:1784949117.364967:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00100000:3.0:1784949117.364967:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675392 00000400:00000010:1.0:1784949117.364977:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000100:00000010:0.0:1784949117.364978:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000200:1.0:1784949117.365007:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289675520, offset 0 00000100:00000001:0.0:1784949117.365010:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949117.365020:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306056747 00000400:00000200:1.0:1784949117.365025:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000010:0.0:1784949117.365029:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:0.0:1784949117.365038:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000400:00000200:1.0:1784949117.365043:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949117.365045:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306056747 00000100:00000010:0.0:1784949117.365053:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000400:00000200:1.0:1784949117.365054:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784949117.365061:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:3.0:1784949117.365064:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:1.0:1784949117.365069:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949117.365070:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949117.365071:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949117.365079:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.365080:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949117.365081:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1784949117.365092:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.365093:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949117.365095:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784949117.365105:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949117.365106:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675392 00000400:00000200:1.0:1784949117.365113:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3639:3639:240:4294967295] 192.168.202.137@tcp LPNI seq info [2336:2336:8:4294967295] 00000020:00000001:3.0:1784949117.365114:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949117.365119:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1784949117.365120:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949117.365126:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:3.0:1784949117.365127:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949117.365136:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1784949117.365138:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949117.365140:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949117.365147:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:3.0:1784949117.365148:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949117.365149:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949117.365161:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949117.365163:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949117.365171:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949117.365174:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000010:1.0:1784949117.365180:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000be1f3156. 00000020:00000010:3.0:1784949117.365182:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000001:3.0:1784949117.365189:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949117.365196:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000800:00000200:1.0:1784949117.365197:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000010:3.0:1784949117.365208:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e6d8b8ad. 00000800:00000200:1.0:1784949117.365233:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000010:3.0:1784949117.365237:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b0a11306. 00000800:00000200:1.0:1784949117.365241:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000be1f3156 type 1, nob 320 niov 1 nkiov 1 00000020:00000010:3.0:1784949117.365248:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000040:3.0:1784949117.365264:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949117.365265:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949117.365269:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:1.0:1784949117.365275:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2182 usecs (3 jiffies) 00000100:00000001:3.0:1784949117.365276:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949117.365281:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.365284:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306056708 00000100:00000001:1.0:1784949117.365289:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306056746 00000100:00000001:1.0:1784949117.365293:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306056708 00000100:00000001:3.0:1784949117.365294:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.365296:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306056747 00000100:00000001:1.0:1784949117.365304:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306056708 00000100:00000001:1.0:1784949117.365307:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306056747 00000100:00000001:1.0:1784949117.365310:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306056708 00000100:00000001:1.0:1784949117.365314:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306056747 00000100:00000001:1.0:1784949117.365318:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949117.365321:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.365323:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.365333:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.365338:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949117.365346:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949117.365349:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949117.365351:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949117.365357:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1784949117.365365:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 780 00000100:00000001:1.0:1784949117.365370:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949117.365370:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949117.365373:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949117.365378:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949117.365379:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949117.365380:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782163024 : -115538927388592 : ffff96eaff467850) 00000100:00000001:2.0:1784949117.365386:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.365389:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949117.365394:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949117.365396:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009bc35acb x1871651289675392/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:23/0 lens 224/0 e 0 to 0 dl 1784949128 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949117.365401:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949117.365405:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949117.365417:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000be1f3156. 00000100:00000001:3.0:1784949117.365420:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949117.365425:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949117.365427:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000001:3.0:1784949117.365435:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949117.365438:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949117.365442:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949117.365444:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000400:00000200:0.0:1784949117.365454:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000800:00000001:2.0:1784949117.365455:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949117.365463:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000100:00100000:3.0:1784949117.365463:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009bc35acb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289675392:12345-0@lo:400: 00000800:00000001:2.0:1784949117.365465:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949117.365472:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949117.365476:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949117.365483:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675392 00000100:00000200:0.0:1784949117.365491:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949117.365493:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:2.0:1784949117.365496:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:3.0:1784949117.365502:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949117.365525:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949117.365536:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949117.365538:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949117.365540:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b700 00000020:00000001:3.0:1784949117.365542:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000040:0.0:1784949117.365550:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949117.365553:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949117.365559:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949117.365562:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 170384 00000020:00000001:3.0:1784949117.365569:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949117.365575:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:2.0:1784949117.365581:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949117.365582:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.365583:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949117.365606:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949117.365609:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784949117.365609:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:2.0:1784949117.365613:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949117.365619:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:2.0:1784949117.365624:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784949117.365629:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949117.365634:0:337114:0:(events.c:286:request_in_callback()) Process entered 02000000:00000010:3.0:1784949117.365638:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009f7bccfc. 00000100:00000200:2.0:1784949117.365639:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000001:3.0:1784949117.365648:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949117.365652:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003dcd6fdf. 00000100:00000001:3.0:1784949117.365654:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949117.365663:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000003dcd6fdf x1871651289675520 msgsize 224 00000020:00000001:3.0:1784949117.365664:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949117.365675:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675392 00000100:00100000:2.0:1784949117.365702:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:3.0:1784949117.365703:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949117.365712:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009bc35acb x1871651289675392/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:23/0 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949117.365722:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949117.365723:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000001:3.0:1784949117.365725:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949117.365730:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=139 v=5 (1 1 1 1) 00000800:00000001:2.0:1784949117.365736:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949117.365738:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949117.365744:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:2.0:1784949117.365746:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949117.365749:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949117.365755:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949117.365757:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949117.365760:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949117.365764:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949117.365770:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00000001:1.0:1784949117.365775:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:3.0:1784949117.365776:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289675392, offset 224 00000100:00100000:1.0:1784949117.365782:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675520 00000400:00000200:3.0:1784949117.365789:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784949117.365790:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949117.365795:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949117.365798:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949117.365802:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949117.365807:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675520 00000400:00000200:3.0:1784949117.365811:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:1.0:1784949117.365813:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949117.365817:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949117.365823:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949117.365826:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949117.365829:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949117.365836:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784949117.365838:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b680 00000100:00000001:1.0:1784949117.365842:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949117.365851:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:1.0:1784949117.365857:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949117.365859:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8f01 [1] + 224 00000020:00000010:1.0:1784949117.365861:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000020:00000010:1.0:1784949117.365870:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006841dfc4. 00000400:00000200:3.0:1784949117.365878:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1784949117.365879:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000169a7180. 00000020:00000010:1.0:1784949117.365885:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000400:00000200:3.0:1784949117.365895:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:1.0:1784949117.365896:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949117.365908:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949117.365919:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949117.365936:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949117.365937:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949117.365942:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949117.365948:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.365957:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949117.365960:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949117.365961:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.365976:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.366014:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949117.366017:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949117.366023:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949117.366026:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949117.366030:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949117.366031:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 756 00000100:00000040:1.0:1784949117.366039:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000200:3.0:1784949117.366042:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000100:00000001:1.0:1784949117.366043:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562014976 : -115544147536640 : ffff96e9c8215500) 00000100:00000001:2.0:1784949117.366045:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949117.366048:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:2.0:1784949117.366056:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949117.366059:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:1.0:1784949117.366060:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003dcd6fdf x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:23/0 lens 224/0 e 0 to 0 dl 1784949128 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949117.366066:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949117.366076:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.366078:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:3.0:1784949117.366084:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000010:1.0:1784949117.366085:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000001:0.0:1784949117.366090:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1784949117.366092:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949117.366094:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949117.366096:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949117.366098:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949117.366102:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000020:00000001:3.0:1784949117.366104:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:0.0:1784949117.366109:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1784949117.366117:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009bc35acb x1871651289675392/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:23/0 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949117.366118:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:1.0:1784949117.366118:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003dcd6fdf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289675520:12345-192.168.202.137@tcp:400: 00000400:00000010:0.0:1784949117.366125:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ee18875c. 00000400:00000200:0.0:1784949117.366137:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000100:00000200:1.0:1784949117.366137:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675520 00000400:00000010:0.0:1784949117.366145:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000020:00000001:1.0:1784949117.366147:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:3.0:1784949117.366154:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009bc35acb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289675392:12345-0@lo:400: Request processed in 727us (1426us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949117.366155:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1784949117.366155:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949117.366163:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949117.366168:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949117.366172:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949117.366179:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00100000:3.0:1784949117.366182:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 780 00000020:00000001:1.0:1784949117.366189:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000040:3.0:1784949117.366193:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:1.0:1784949117.366196:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949117.366203:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:1.0:1784949117.366205:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949117.366209:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:0.0:1784949117.366228:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949117.366228:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949117.366236:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949117.366236:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b0a11306. 00000020:00000001:1.0:1784949117.366248:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000010:3.0:1784949117.366251:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949117.366258:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949117.366259:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000010:3.0:1784949117.366260:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e6d8b8ad. 00000100:00000001:0.0:1784949117.366268:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949117.366269:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000040:3.0:1784949117.366271:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000010:1.0:1784949117.366278:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008905542c. 00000100:00000001:0.0:1784949117.366279:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949117.366281:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:0.0:1784949117.366286:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:1.0:1784949117.366288:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949117.366289:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009f7bccfc. 00000100:00000001:0.0:1784949117.366290:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949117.366298:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.366299:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949117.366301:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949117.366308:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949117.366310:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949117.366310:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949117.366316:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000040:1.0:1784949117.366324:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675520 00000100:00001000:0.0:1784949117.366326:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=139 v=5 (5 5 5 5) 00000400:00000010:3.0:1784949117.366328:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000037f9e020. 00010000:00000001:1.0:1784949117.366335:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00001000:0.0:1784949117.366343:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=139 v=5 (1 1 1 1) 00000400:00000010:3.0:1784949117.366343:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007e436012. 00010000:00000200:1.0:1784949117.366353:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003dcd6fdf x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:23/0 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949117.366356:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949117.366361:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949117.366369:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784949117.366376:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949117.366386:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949117.366389:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:0.0:1784949117.366395:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949117.366397:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949117.366410:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=134 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949117.366414:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949117.366423:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949117.366423:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949117.366430:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949117.366434:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000040:0.0:1784949117.366441:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949117.366442:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949117.366450:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949117.366454:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.366462:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949117.366471:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289675392:0@lo:400: 00000100:00000040:1.0:1784949117.366473:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949117.366483:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949117.366484:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000100:00000001:0.0:1784949117.366490:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1784949117.366495:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289675520, offset 224 00000100:00000040:0.0:1784949117.366498:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289675392/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949117.366528:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949117.366540:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1784949117.366546:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:0.0:1784949117.366547:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949117.366553:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 00000400:00000200:1.0:1784949117.366561:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3640:3640:240:4294967295] 192.168.202.137@tcp LPNI seq info [2337:2337:8:4294967295] 02000000:00000001:0.0:1784949117.366565:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949117.366569:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949117.366575:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949117.366583:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:1.0:1784949117.366584:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000010:0.0:1784949117.366589:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 00000800:00000200:1.0:1784949117.366613:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784949117.366614:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949117.366619:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000010:1.0:1784949117.366621:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000be1f3156. 00000100:00000010:0.0:1784949117.366623:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000100:00000001:0.0:1784949117.366632:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949117.366636:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949117.366637:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949117.366641:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949117.366650:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:0.0:1784949117.366651:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949117.366657:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949117.366658:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000be1f3156 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949117.366663:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949117.366668:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.366681:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949117.366688:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949117.366691:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949117.366693:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949117.366696:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949117.366701:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003dcd6fdf x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:23/0 lens 224/224 e 0 to 0 dl 1784949128 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949117.366713:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003dcd6fdf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289675520:12345-192.168.202.137@tcp:400: Request processed in 635us (1049us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949117.366738:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 756 00000100:00000040:1.0:1784949117.366742:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949117.366745:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949117.366747:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949117.366750:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000169a7180. 00000020:00000010:1.0:1784949117.366755:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949117.366759:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006841dfc4. 00000020:00000040:1.0:1784949117.366763:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949117.366765:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949117.366895:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949117.366902:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000be1f3156. 00000800:00000001:0.0:1784949117.366935:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949117.366943:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949117.366949:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949117.366972:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949117.366974:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949117.367001:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949117.367013:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000200:0.0:1784949117.367014:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:3.0:1784949117.367046:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000100:00000001:3.0:1784949117.367053:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949117.367056:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784949117.367061:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b700 00000400:00000200:0.0:1784949117.367071:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8f09 [1] + 224 00000800:00000001:0.0:1784949117.367078:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949117.367091:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949117.367094:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949117.367098:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949117.367103:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949117.367107:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949117.367117:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949117.367141:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949117.367148:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949117.367150:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.367173:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949117.367178:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949117.367183:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949117.367185:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949117.367187:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ba749c18. 00000400:00000200:1.0:1784949117.367193:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784949117.367195:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:1.0:1784949117.367198:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949117.367204:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949117.367214:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949117.367222:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949117.367224:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.367244:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949117.367247:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949117.367249:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949117.367250:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949117.367253:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949117.367259:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=134 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949117.367262:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=134 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949117.367265:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949117.367267:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949117.367269:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949117.367273:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949117.367282:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.367286:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949117.367288:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949117.367291:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949117.367292:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949117.367296:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949117.367304:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949117.367306:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949117.367308:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949117.367313:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949117.367327:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000031b67b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289675520:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949117.367335:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.367338:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949117.367342:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000031b67b62 x1871651289675520/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949117.367355:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949117.367358:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949117.367361:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 02000000:00000001:1.0:1784949117.367368:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949117.367371:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949117.367373:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949117.367376:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949117.367380:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:1.0:1784949117.367385:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949117.367388:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949117.367390:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000031b67b62. 00000100:00000001:1.0:1784949117.367396:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949117.367398:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.367401:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949117.367408:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949117.367411:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949117.367414:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949117.367420:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949117.408063:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306056746, at 4306056748 00000100:00000010:0.0:1784949117.408080:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000040:1.0:1784949117.408090:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306056747, at 4306056748 00000100:00000040:2.0:1784949117.408101:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306056747, at 4306056748 00000100:00000010:1.0:1784949117.408105:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000010:2.0:1784949117.408115:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:3.0:1784949117.408130:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306056747, at 4306056748 00000100:00000010:3.0:1784949117.408145:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00010000:00000001:2.0:1784949118.196042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949118.196052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949118.196061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949118.196063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949118.196159:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949118.196166:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949118.196172:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949118.388040:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949118.388056:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949118.388058:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949118.388063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949118.388066:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949118.388070:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949118.388073:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949119.220029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949119.220037:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949119.220044:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949119.220045:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949119.220107:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949119.220113:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949119.220116:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949119.412006:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949119.412019:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949119.412021:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949119.412025:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949119.412027:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949119.412031:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949119.412034:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949120.244038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949120.244047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949120.244054:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949120.244055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949120.244179:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949120.244186:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949120.244191:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949120.436128:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949120.436144:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949120.436146:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949120.436152:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949120.436156:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949120.436160:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949120.436163:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949121.268019:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949121.268028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949121.268036:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949121.268040:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949121.268043:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949121.268046:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949121.268050:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949121.268053:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949121.268055:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949121.268058:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949121.268180:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949121.268188:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949121.268192:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949121.460063:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949121.460084:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949121.460089:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949121.460098:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949121.460103:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949121.460108:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949121.460111:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949122.292054:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949122.292064:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949122.292072:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949122.292074:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949122.292171:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949122.292178:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949122.292183:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949122.484104:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949122.484117:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949122.484127:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000020:00000040:0.0:1784949122.484136:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949122.484144:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949122.484148:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949122.484155:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949122.484167:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949122.484178:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949122.484189:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949122.484205:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949122.484208:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949122.484216:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_00_03+3] req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949122.484227:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949122.484245:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949122.484249:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949122.484255:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949122.484259:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949122.484264:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949122.484267:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949122.484269:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949122.484275:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:1.0:1784949122.484277:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784949122.484286:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1784949122.484287:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000040:0.0:1784949122.484293:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949122.484295:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949122.484306:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949122.484309:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1784949122.484311:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000617daa1a. 00000100:00000001:1.0:1784949122.484316:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949122.484318:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949122.484324:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949122.484326:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949122.484326:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949122.484330:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949122.484334:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949122.484337:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949122.484338:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949122.484339:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949122.484351:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949122.484353:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002a753d9d. 00000100:00000001:0.0:1784949122.484363:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949122.484377:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949122.484378:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949122.484384:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949122.484394:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949122.484399:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949122.484401:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949122.484403:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949122.484414:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949122.484418:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000617daa1a] to pc [ptlrpcd_00_00+0] req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949122.484426:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289675648:0@lo:400: 00000100:00000001:1.0:1784949122.484444:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949122.484454:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949122.484457:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949122.484461:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949122.484466:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949122.484469:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 02000000:00000001:1.0:1784949122.484474:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949122.484475:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11399) 00000400:00000010:1.0:1784949122.484484:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba749c18. 00000400:00000010:1.0:1784949122.484496:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000001:3.0:1784949122.484500:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1784949122.484506:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675648, portal 25 00000100:00100000:3.0:1784949122.484508:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:2.0:1784949122.484512:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949122.484515:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949122.484517:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949122.484520:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 00000100:00000001:2.0:1784949122.484521:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949122.484525:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949122.484533:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949122.484538:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949122.484542:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949122.484548:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949122.484553:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949122.484558:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949122.484569:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949122.484571:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949122.484579:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000200:1.0:1784949122.484587:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289675648, offset 0 00000100:00000001:3.0:1784949122.484601:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784949122.484603:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949122.484608:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949122.484614:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949122.484617:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949122.484620:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949122.484630:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289675776:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784949122.484634:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949122.484649:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949122.484655:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949122.484660:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949122.484670:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b780 02000000:00000001:3.0:1784949122.484671:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949122.484684:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000bbdbdf4. 00000400:00000200:1.0:1784949122.484693:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8511 [2] + 0 02000000:00000001:3.0:1784949122.484694:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949122.484708:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001db3b9dd. 00000400:00000010:1.0:1784949122.484710:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bb87e206. 00000400:00000200:1.0:1784949122.484724:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d3baa36a 00000400:00000010:3.0:1784949122.484735:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000400:00000200:1.0:1784949122.484739:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:3.0:1784949122.484746:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675776, portal 10 00000100:00000001:3.0:1784949122.484754:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949122.484756:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949122.484758:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554966272 : -115544154585344 : ffff96e9c7b5c700) 00000400:00000200:1.0:1784949122.484774:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3baa36a 00000100:00000040:3.0:1784949122.484775:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949122.484785:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d3baa36a. 00000100:00000001:3.0:1784949122.484797:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949122.484803:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784949122.484803:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949122.484812:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000100:00000200:1.0:1784949122.484814:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:3.0:1784949122.484817:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289675776, offset 0 00000100:00000040:1.0:1784949122.484828:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000078cfc26 x1871651289675648 msgsize 224 00000400:00000200:3.0:1784949122.484829:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784949122.484846:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784949122.484848:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:1.0:1784949122.484856:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949122.484860:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949122.484871:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949122.484875:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949122.484880:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949122.484884:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3641:3641:240:4294967295] 192.168.202.137@tcp LPNI seq info [2338:2338:8:4294967295] 00000100:00000001:1.0:1784949122.484891:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949122.484934:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949122.484966:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949122.484973:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784949122.484984:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949122.484998:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:1.0:1784949122.485001:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000800:00000200:3.0:1784949122.485004:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000400:00000010:1.0:1784949122.485011:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:1.0:1784949122.485022:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949122.485024:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949122.485026:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:1.0:1784949122.485034:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949122.485036:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675648 00000100:00000040:3.0:1784949122.485037:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2183 usecs (3 jiffies) 02000000:00000001:2.0:1784949122.485045:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949122.485049:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306061828 00000100:00000001:2.0:1784949122.485052:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949122.485059:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949122.485064:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 02000000:00000001:2.0:1784949122.485071:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949122.485080:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00100000:2.0:1784949122.485082:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675648 00000100:00000001:1.0:1784949122.485091:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949122.485092:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949122.485095:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306061828 00000020:00000040:2.0:1784949122.485099:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:1.0:1784949122.485104:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784949122.485105:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000020:00000001:2.0:1784949122.485107:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949122.485116:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000040:2.0:1784949122.485117:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949122.485128:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306061828 00000020:00000001:2.0:1784949122.485129:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784949122.485139:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000029366a5. 00000020:00000001:2.0:1784949122.485141:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949122.485145:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949122.485149:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949122.485156:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949122.485157:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949122.485162:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306061828 00000020:00000010:2.0:1784949122.485168:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006a06b5eb. 00000100:00000010:3.0:1784949122.485172:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000100:00000001:1.0:1784949122.485173:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949122.485176:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949122.485177:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949122.485181:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949122.485188:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306061867 00000800:00000010:0.0:1784949122.485190:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000020:00000010:2.0:1784949122.485190:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006a06b5eb. 00000100:00000001:3.0:1784949122.485193:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949122.485205:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949122.485207:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949122.485210:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949122.485212:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306061828 00000020:00000010:2.0:1784949122.485212:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006b8bfa56. 00000100:00000001:3.0:1784949122.485219:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949122.485225:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:2.0:1784949122.485226:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003f674c5f. 00000100:00000001:1.0:1784949122.485235:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306061867 00000400:00000200:0.0:1784949122.485240:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000020:00000010:2.0:1784949122.485241:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000010:0.0:1784949122.485250:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 00000100:00000001:1.0:1784949122.485250:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306061828 00000800:00000001:3.0:1784949122.485251:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949122.485259:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949122.485265:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949122.485268:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306061867 00000100:00000001:2.0:1784949122.485269:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000800:00000001:3.0:1784949122.485271:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949122.485276:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:0.0:1784949122.485277:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949122.485283:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949122.485285:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949122.485287:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306061828 00000800:00000001:3.0:1784949122.485294:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949122.485304:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306061867 00000100:00000001:2.0:1784949122.485305:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949122.485321:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949122.485323:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306061828 00000100:00000001:2.0:1784949122.485335:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949122.485338:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949122.485349:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949122.485353:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949122.485354:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b800 00000100:00000001:0.0:1784949122.485356:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949122.485357:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784949122.485369:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949122.485371:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949122.485378:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 781 00000400:00000200:3.0:1784949122.485383:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 170608 00000100:00000001:1.0:1784949122.485387:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949122.485391:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949122.485403:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782150736 : -115538927400880 : ffff96eaff464850) 00000800:00000001:3.0:1784949122.485408:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949122.485418:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949122.485420:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000078cfc26 x1871651289675648/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:28/0 lens 224/0 e 0 to 0 dl 1784949133 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949122.485422:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949122.485424:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784949122.485435:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949122.485448:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949122.485449:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:2.0:1784949122.485459:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006a06b5eb. 00000100:00000001:3.0:1784949122.485462:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949122.485468:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784949122.485470:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949122.485478:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006a06b5eb. 00000100:00000010:3.0:1784949122.485480:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005c8e6480. 00000100:00100000:2.0:1784949122.485489:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000078cfc26 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289675648:12345-0@lo:400: 00000100:00000040:3.0:1784949122.485490:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000005c8e6480 x1871651289675776 msgsize 224 00000100:00000200:2.0:1784949122.485505:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675648 00000100:00100000:3.0:1784949122.485511:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:2.0:1784949122.485514:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949122.485521:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949122.485524:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949122.485527:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949122.485530:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949122.485534:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:2.0:1784949122.485535:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949122.485541:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949122.485545:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:3.0:1784949122.485548:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949122.485549:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949122.485557:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949122.485558:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949122.485562:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949122.485568:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949122.485571:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949122.485578:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000032069f11. 02000000:00000001:2.0:1784949122.485583:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949122.485587:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949122.485590:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949122.485594:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675648 00010000:00000001:2.0:1784949122.485598:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949122.485606:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000078cfc26 x1871651289675648/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:28/0 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949122.485617:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:2.0:1784949122.485618:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949122.485625:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949122.485627:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675776 00000100:00001000:2.0:1784949122.485632:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=144 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949122.485637:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949122.485640:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949122.485644:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:2.0:1784949122.485647:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949122.485652:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949122.485654:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784949122.485664:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949122.485666:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949122.485667:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949122.485678:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949122.485681:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675776 00000100:00000040:2.0:1784949122.485683:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:0.0:1784949122.485691:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:2.0:1784949122.485693:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000020:00000040:0.0:1784949122.485697:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000200:2.0:1784949122.485701:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289675648, offset 224 00000020:00000001:0.0:1784949122.485706:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1784949122.485714:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000040:0.0:1784949122.485716:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949122.485727:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784949122.485729:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:0.0:1784949122.485739:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784949122.485745:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b780 00000100:00000001:0.0:1784949122.485753:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784949122.485758:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8f1d [1] + 224 00000020:00000010:0.0:1784949122.485767:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000400:00000200:2.0:1784949122.485770:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949122.485776:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949122.485783:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:0.0:1784949122.485785:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:2.0:1784949122.485793:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784949122.485796:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000200:2.0:1784949122.485802:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949122.485807:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000150ceb7b. 00000020:00000010:0.0:1784949122.485817:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:2.0:1784949122.485828:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949122.485831:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949122.485839:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949122.485844:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949122.485851:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949122.485859:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949122.485862:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949122.485864:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949122.485870:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949122.485880:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000100:00000001:0.0:1784949122.485883:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949122.485884:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000100:00000001:2.0:1784949122.485895:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949122.485900:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949122.485961:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949122.485968:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949122.485976:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949122.485977:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949122.485983:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 757 00000100:00000001:2.0:1784949122.485984:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949122.485985:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949122.485986:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949122.485999:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000001:1.0:1784949122.485999:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784949122.486001:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949122.486002:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949122.486006:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949122.486011:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554963328 : -115544154588288 : ffff96e9c7b5bb80) 00000100:00000001:3.0:1784949122.486017:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949122.486018:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ba749c18. 00010000:00000001:2.0:1784949122.486019:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949122.486030:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949122.486031:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005c8e6480 x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:28/0 lens 224/0 e 0 to 0 dl 1784949133 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949122.486032:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784949122.486041:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000020:00000001:2.0:1784949122.486043:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949122.486052:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1784949122.486061:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000078cfc26 x1871651289675648/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:28/0 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949122.486064:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949122.486065:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949122.486074:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:0.0:1784949122.486082:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949122.486091:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00100000:0.0:1784949122.486111:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005c8e6480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289675776:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949122.486132:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949122.486134:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675776 00000020:00000001:0.0:1784949122.486142:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:2.0:1784949122.486148:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000078cfc26 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289675648:12345-0@lo:400: Request processed in 689us (1309us total) trans 0 rc 0/0 00000020:00000001:0.0:1784949122.486151:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949122.486158:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949122.486163:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1784949122.486167:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949122.486169:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949122.486174:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784949122.486178:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:0.0:1784949122.486180:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 02000000:00000001:1.0:1784949122.486184:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:0.0:1784949122.486188:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949122.486190:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 781 00000100:00000001:1.0:1784949122.486191:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:0.0:1784949122.486197:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784949122.486198:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949122.486206:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:0.0:1784949122.486207:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:1.0:1784949122.486208:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949122.486215:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949122.486220:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=144 v=5 (5 5 5 5) 00000100:00000001:2.0:1784949122.486220:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:0.0:1784949122.486221:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949122.486230:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949122.486232:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00001000:1.0:1784949122.486233:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=144 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949122.486241:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949122.486243:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:2.0:1784949122.486245:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003f674c5f. 00000100:00000001:1.0:1784949122.486248:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949122.486249:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007f1baa02. 00000020:00000010:2.0:1784949122.486260:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 02000000:00000001:0.0:1784949122.486261:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949122.486262:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949122.486268:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949122.486271:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949122.486275:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949122.486277:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949122.486277:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006b8bfa56. 00000100:00000040:1.0:1784949122.486287:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:0.0:1784949122.486288:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675776 00010000:00000001:0.0:1784949122.486298:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000040:2.0:1784949122.486300:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000200:0.0:1784949122.486311:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005c8e6480 x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:28/0 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949122.486315:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949122.486318:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949122.486326:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:2.0:1784949122.486327:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000032069f11. 00000100:00000001:1.0:1784949122.486333:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949122.486339:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:0.0:1784949122.486341:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949122.486345:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949122.486347:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949122.486348:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949122.486359:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:0.0:1784949122.486360:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=139 v=5 (1 1 1 1) 00000400:00000010:2.0:1784949122.486374:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba88f347. 00000100:00000001:0.0:1784949122.486377:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1784949122.486379:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289675648:0@lo:400: 00000400:00000010:2.0:1784949122.486389:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000089ea3182. 00000100:00000040:0.0:1784949122.486390:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949122.486393:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949122.486401:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:1.0:1784949122.486403:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949122.486411:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:1.0:1784949122.486413:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289675648/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949122.486415:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949122.486425:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949122.486435:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949122.486437:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000010:0.0:1784949122.486443:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 02000000:00000001:1.0:1784949122.486443:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949122.486449:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 00000100:00000200:0.0:1784949122.486452:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289675776, offset 224 02000000:00000001:1.0:1784949122.486456:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949122.486462:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949122.486466:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:0.0:1784949122.486467:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949122.486475:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949122.486479:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:1.0:1784949122.486484:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:0.0:1784949122.486489:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:1.0:1784949122.486490:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949122.486496:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 00000400:00000200:0.0:1784949122.486504:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3642:3642:240:4294967295] 192.168.202.137@tcp LPNI seq info [2339:2339:8:4294967295] 00000100:00000001:1.0:1784949122.486512:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949122.486519:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949122.486528:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949122.486530:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949122.486537:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:0.0:1784949122.486543:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949122.486544:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949122.486552:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000820f02b3. 00000100:00000001:1.0:1784949122.486553:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949122.486560:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949122.486567:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949122.486576:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949122.486581:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000820f02b3 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949122.486613:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949122.486617:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949122.486620:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949122.486622:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949122.486624:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949122.486628:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005c8e6480 x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:28/0 lens 224/224 e 0 to 0 dl 1784949133 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949122.486640:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005c8e6480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289675776:12345-192.168.202.137@tcp:400: Request processed in 576us (1149us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949122.486653:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 757 00000100:00000040:0.0:1784949122.486658:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949122.486661:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949122.486663:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949122.486667:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000150ceb7b. 00000020:00000010:0.0:1784949122.486672:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949122.486677:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000020:00000040:0.0:1784949122.486684:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949122.486687:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949122.486818:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949122.486826:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000820f02b3. 00000800:00000001:0.0:1784949122.486828:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949122.486838:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949122.486843:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949122.486850:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949122.486856:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949122.486857:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949122.486871:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000200:0.0:1784949122.486873:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:1.0:1784949122.486881:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00000001:1.0:1784949122.486892:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:0.0:1784949122.486894:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b800 00000100:00000001:1.0:1784949122.486900:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784949122.486932:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8f25 [1] + 224 00000800:00000001:0.0:1784949122.486961:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949122.486980:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949122.486986:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949122.486994:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949122.487003:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949122.487011:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949122.487032:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949122.487055:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949122.487063:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949122.487067:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949122.487124:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949122.487130:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949122.487137:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949122.487139:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949122.487142:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001db3b9dd. 00000400:00000200:3.0:1784949122.487151:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:3.0:1784949122.487153:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:3.0:1784949122.487157:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949122.487165:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949122.487178:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949122.487188:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949122.487190:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949122.487193:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949122.487196:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949122.487199:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949122.487201:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949122.487204:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949122.487211:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=139 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949122.487215:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=139 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949122.487218:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949122.487220:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949122.487223:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949122.487227:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949122.487236:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949122.487240:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949122.487242:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949122.487245:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949122.487247:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949122.487251:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949122.487259:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949122.487262:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949122.487263:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949122.487267:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949122.487275:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289675776:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949122.487280:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949122.487282:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949122.487284:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000617daa1a x1871651289675776/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949122.487293:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949122.487295:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949122.487298:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000bbdbdf4. 02000000:00000001:3.0:1784949122.487302:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949122.487304:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949122.487306:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949122.487309:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949122.487311:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002a753d9d. 02000000:00000001:3.0:1784949122.487314:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949122.487315:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949122.487317:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000617daa1a. 00000100:00000001:3.0:1784949122.487327:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949122.487328:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949122.487330:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949122.487335:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949122.487337:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949122.487340:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949122.487342:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949122.528033:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306061867, at 4306061868 00000100:00000040:3.0:1784949122.528048:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306061867, at 4306061868 00000100:00000010:2.0:1784949122.528050:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000029366a5. 00000100:00000040:0.0:1784949122.528061:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306061867, at 4306061868 00000100:00000040:1.0:1784949122.528065:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306061867, at 4306061868 00000100:00000010:3.0:1784949122.528067:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000100:00000010:0.0:1784949122.528084:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000010:1.0:1784949122.528088:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00010000:00000001:2.0:1784949123.316074:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949123.316084:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949123.316092:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949123.316094:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949123.316218:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949123.316225:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949123.316230:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949123.508081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949123.508096:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949123.508098:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949123.508103:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949123.508106:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949123.508110:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949123.508113:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949124.340030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949124.340038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949124.340043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949124.340044:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949124.340124:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949124.340131:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949124.340135:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949124.532172:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949124.532189:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949124.532192:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949124.532199:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949124.532203:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949124.532208:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949124.532211:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949125.364046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949125.364055:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949125.364062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949125.364064:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949125.364166:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949125.364176:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949125.364184:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949125.555995:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949125.556006:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949125.556007:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949125.556011:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949125.556014:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949125.556017:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949125.556020:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949126.388058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949126.388067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949126.388075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949126.388077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949126.388151:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949126.388157:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949126.388160:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949126.580049:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949126.580063:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949126.580066:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949126.580072:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949126.580076:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949126.580081:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949126.580084:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949127.412067:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949127.412075:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949127.412083:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949127.412085:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949127.412151:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949127.412158:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949127.412163:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949127.604054:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949127.604078:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784949127.604087:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000010:0.0:1784949127.604088:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949127.604099:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949127.604103:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949127.604106:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784949127.604109:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949127.604113:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949127.604118:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949127.604124:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.604124:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949127.604133:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.604138:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949127.604140:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005a1c05a3. 00000100:00000001:0.0:1784949127.604151:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949127.604164:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949127.604190:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949127.604193:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949127.604200:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_01+1] req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949127.604233:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949127.604236:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949127.604241:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949127.604244:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949127.604249:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949127.604253:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949127.604257:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949127.604260:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949127.604262:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949127.604264:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.604267:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949127.604270:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002a753d9d. 00000100:00000001:3.0:1784949127.604275:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949127.604278:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.604281:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949127.604288:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949127.604293:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949127.604294:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949127.604303:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949127.604334:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949127.604341:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.604342:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949127.604349:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949127.604355:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_02+2] req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949127.604357:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949127.604366:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949127.604376:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949127.604384:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289675904:0@lo:400: 00000100:00000001:3.0:1784949127.604391:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949127.604397:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949127.604399:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949127.604407:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949127.604408:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11404) 02000000:00000001:3.0:1784949127.604411:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949127.604422:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000229221cc. 02000000:00000001:3.0:1784949127.604426:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949127.604435:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001db3b9dd. 00000100:00000001:2.0:1784949127.604440:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949127.604445:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00100000:2.0:1784949127.604447:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:2.0:1784949127.604455:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949127.604460:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289675904, portal 25 00000100:00000001:2.0:1784949127.604463:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949127.604469:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949127.604470:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949127.604474:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000100:00000001:2.0:1784949127.604477:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949127.604483:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784949127.604488:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949127.604489:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949127.604503:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949127.604505:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949127.604508:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949127.604512:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949127.604522:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949127.604531:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00000200:3.0:1784949127.604541:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289675904, offset 0 00000100:00000001:2.0:1784949127.604561:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1784949127.604561:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949127.604568:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949127.604573:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949127.604576:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949127.604580:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949127.604588:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289676032:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784949127.604591:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949127.604603:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949127.604609:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949127.604612:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949127.604618:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784949127.604619:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b880 02000000:00000010:2.0:1784949127.604628:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006b8bfa56. 00000400:00000200:3.0:1784949127.604636:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8535 [2] + 0 02000000:00000001:2.0:1784949127.604638:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949127.604650:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000069afdf3e. 00000400:00000010:2.0:1784949127.604652:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001712b5b2. 00000400:00000200:3.0:1784949127.604657:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b3c5fd98 00000400:00000200:3.0:1784949127.604669:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1784949127.604675:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000400:00000200:3.0:1784949127.604676:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:2.0:1784949127.604688:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289676032, portal 10 00000400:00000200:3.0:1784949127.604692:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b3c5fd98 00000100:00000001:2.0:1784949127.604696:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:3.0:1784949127.604698:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b3c5fd98. 00000100:00000001:2.0:1784949127.604700:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000100:00000001:3.0:1784949127.604706:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949127.604711:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949127.604716:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949127.604719:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000d9986ea8 x1871651289675904 msgsize 224 00000100:00100000:3.0:1784949127.604740:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784949127.604747:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949127.604751:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949127.604760:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949127.604763:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000100:00000200:2.0:1784949127.604767:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289676032, offset 0 00000100:00000001:3.0:1784949127.604774:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949127.604776:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949127.604790:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000400:00000010:3.0:1784949127.604795:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000400:00000200:2.0:1784949127.604801:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949127.604804:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949127.604812:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:3.0:1784949127.604815:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949127.604825:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949127.604833:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784949127.604839:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00100000:1.0:1784949127.604843:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289675904 00000100:00000001:3.0:1784949127.604844:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949127.604849:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:1.0:1784949127.604855:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:3.0:1784949127.604859:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949127.604863:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784949127.604865:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3643:3643:240:4294967295] 192.168.202.137@tcp LPNI seq info [2340:2340:8:4294967295] 00000100:00000001:1.0:1784949127.604870:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949127.604883:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949127.604946:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289675904 00000020:00000001:1.0:1784949127.604952:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949127.604955:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949127.604964:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949127.604972:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949127.604974:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:2.0:1784949127.604980:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949127.604984:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784949127.604986:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784949127.604993:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949127.605000:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949127.605002:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949127.605005:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949127.605011:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306066987 00000800:00000010:2.0:1784949127.605014:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005523afda. 00000020:00000010:1.0:1784949127.605016:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000d6df185. 00000100:00000010:3.0:1784949127.605023:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000100:00000001:1.0:1784949127.605025:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1784949127.605035:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000010:1.0:1784949127.605037:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000d6df185. 00000100:00000001:3.0:1784949127.605039:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:1.0:1784949127.605054:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006841dfc4. 00000800:00000200:2.0:1784949127.605056:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784949127.605062:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306066987 00000800:00000200:2.0:1784949127.605066:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005523afda type 1, nob 320 niov 1 nkiov 1 00000020:00000010:1.0:1784949127.605067:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000169a7180. 00000100:00000010:3.0:1784949127.605074:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000020:00000010:1.0:1784949127.605081:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949127.605084:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949127.605094:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.605096:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306066987 00000100:00000040:1.0:1784949127.605100:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949127.605108:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:2.0:1784949127.605109:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2185 usecs (3 jiffies) 00000100:00000010:3.0:1784949127.605110:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002b896c5a. 00000100:00000001:1.0:1784949127.605112:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949127.605119:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949127.605123:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306066948 00000100:00000001:3.0:1784949127.605123:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949127.605131:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949127.605136:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306066987 00000100:00000001:3.0:1784949127.605137:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306066987 00000100:00000001:2.0:1784949127.605147:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306066948 00000100:00000010:3.0:1784949127.605150:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:1.0:1784949127.605151:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949127.605154:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306066987 00000100:00000001:2.0:1784949127.605163:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306066948 00000100:00000001:3.0:1784949127.605165:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949127.605168:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306066987 00000100:00000001:2.0:1784949127.605174:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306066948 00000100:00000001:3.0:1784949127.605179:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949127.605180:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949127.605180:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306066987 00000100:00000001:1.0:1784949127.605188:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949127.605192:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949127.605194:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949127.605202:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949127.605204:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 782 00000100:00000001:2.0:1784949127.605206:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.605210:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949127.605217:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949127.605219:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.605223:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949127.605225:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782164048 : -115538927387568 : ffff96eaff467c50) 00000800:00000200:0.0:1784949127.605227:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949127.605228:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.605230:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:1.0:1784949127.605239:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d9986ea8 x1871651289675904/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:33/0 lens 224/0 e 0 to 0 dl 1784949138 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949127.605244:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:0.0:1784949127.605245:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005523afda. 00000100:00000001:3.0:1784949127.605251:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949127.605261:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949127.605267:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949127.605269:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949127.605271:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949127.605279:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000d6df185. 00000800:00000001:2.0:1784949127.605283:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949127.605290:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949127.605295:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:1.0:1784949127.605298:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000d6df185. 00000800:00000001:2.0:1784949127.605303:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949127.605310:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949127.605313:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000100:00100000:1.0:1784949127.605315:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d9986ea8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289675904:12345-0@lo:400: 00000800:00000001:2.0:1784949127.605316:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949127.605326:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000100:00000200:1.0:1784949127.605329:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289675904 00000020:00000001:1.0:1784949127.605339:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:2.0:1784949127.605340:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949127.605344:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784949127.605346:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949127.605353:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949127.605358:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643b900 00000100:00000200:0.0:1784949127.605364:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949127.605364:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949127.605371:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:2.0:1784949127.605381:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 170832 00000020:00000001:1.0:1784949127.605383:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949127.605389:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949127.605402:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:2.0:1784949127.605404:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949127.605410:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949127.605415:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949127.605416:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949127.605417:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784949127.605422:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:2.0:1784949127.605427:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949127.605429:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:0.0:1784949127.605435:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949127.605438:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:2.0:1784949127.605446:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000010:1.0:1784949127.605448:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e5630c1a. 02000000:00000001:1.0:1784949127.605461:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949127.605465:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949127.605471:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949127.605473:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949127.605476:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949127.605482:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949127.605486:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b77922d6. 00000100:00000001:0.0:1784949127.605487:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000040:1.0:1784949127.605490:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289675904 00000100:00000040:2.0:1784949127.605496:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000b77922d6 x1871651289676032 msgsize 224 00010000:00000001:1.0:1784949127.605498:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949127.605509:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d9986ea8 x1871651289675904/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:33/0 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949127.605515:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:1.0:1784949127.605529:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949127.605533:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949127.605537:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=149 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949127.605541:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949127.605543:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949127.605550:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949127.605553:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:2.0:1784949127.605555:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784949127.605559:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949127.605565:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949127.605565:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949127.605573:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949127.605580:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949127.605586:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000200:1.0:1784949127.605592:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289675904, offset 224 00000400:00000200:1.0:1784949127.605602:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949127.605603:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949127.605608:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289676032 02000000:00000001:3.0:1784949127.605613:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949127.605619:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784949127.605620:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949127.605625:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949127.605631:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949127.605639:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b880 00000100:00100000:3.0:1784949127.605640:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289676032 00000020:00000001:3.0:1784949127.605650:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949127.605653:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:1.0:1784949127.605656:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8f39 [1] + 224 00000020:00000001:3.0:1784949127.605659:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949127.605666:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:1.0:1784949127.605670:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784949127.605672:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949127.605679:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:1.0:1784949127.605683:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949127.605689:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949127.605692:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949127.605697:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000200:1.0:1784949127.605701:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949127.605705:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949127.605715:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000020:00000010:3.0:1784949127.605727:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000467d4d37. 00000020:00000010:3.0:1784949127.605737:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b0a11306. 00000020:00000010:3.0:1784949127.605747:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000040:1.0:1784949127.605753:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949127.605760:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949127.605768:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949127.605772:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949127.605775:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949127.605783:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949127.605786:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949127.605787:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949127.605790:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784949127.605802:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784949127.605804:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:1.0:1784949127.605809:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949127.605811:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949127.605812:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784949127.605823:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949127.605831:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949127.605833:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00010000:00000001:1.0:1784949127.605836:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949127.605839:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000001:1.0:1784949127.605841:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949127.605847:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949127.605852:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949127.605854:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 758 00000100:00000040:1.0:1784949127.605857:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d9986ea8 x1871651289675904/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:33/0 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949127.605860:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784949127.605866:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000001:0.0:1784949127.605873:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949127.605876:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562013184 : -115544147538432 : ffff96e9c8214e00) 00000400:00000001:0.0:1784949127.605881:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949127.605890:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001db3b9dd. 00000100:00000040:3.0:1784949127.605955:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b77922d6 x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:33/0 lens 224/0 e 0 to 0 dl 1784949138 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949127.605957:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d9986ea8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289675904:12345-0@lo:400: Request processed in 624us (1173us total) trans 0 rc 0/0 00000400:00000200:0.0:1784949127.605965:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:0.0:1784949127.605973:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:0.0:1784949127.605986:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784949127.605993:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 782 00000100:00000001:3.0:1784949127.605993:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:0.0:1784949127.606001:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949127.606003:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000010602010. 00000100:00000040:1.0:1784949127.606004:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949127.606017:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949127.606017:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784949127.606024:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949127.606026:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949127.606027:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000010602010. 00000020:00000010:1.0:1784949127.606038:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000169a7180. 00000100:00100000:3.0:1784949127.606050:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b77922d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289676032:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949127.606053:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:1.0:1784949127.606054:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949127.606061:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949127.606066:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289676032 00000020:00000010:1.0:1784949127.606067:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006841dfc4. 00000100:00000001:0.0:1784949127.606069:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:3.0:1784949127.606075:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:0.0:1784949127.606077:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:1.0:1784949127.606080:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:3.0:1784949127.606082:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949127.606083:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:3.0:1784949127.606090:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949127.606091:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784949127.606092:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949127.606099:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000e5630c1a. 00000020:00000001:3.0:1784949127.606102:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:0.0:1784949127.606104:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949127.606108:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:3.0:1784949127.606109:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00001000:0.0:1784949127.606112:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=149 v=5 (5 5 5 5) 00000100:00000001:1.0:1784949127.606115:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949127.606119:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949127.606126:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784949127.606129:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=149 v=5 (1 1 1 1) 00000100:00000200:1.0:1784949127.606130:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:3.0:1784949127.606131:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949127.606138:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784949127.606144:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784949127.606145:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949127.606146:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba749c18. 00000100:00000001:0.0:1784949127.606151:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949127.606157:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00010000:00000001:0.0:1784949127.606159:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:1.0:1784949127.606160:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000553d6b82. 00000100:00000001:3.0:1784949127.606164:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00010000:00000001:0.0:1784949127.606166:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949127.606172:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949127.606174:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949127.606181:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001499f505. 00000100:00000040:0.0:1784949127.606184:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949127.606190:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949127.606199:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949127.606209:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949127.606212:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000040:3.0:1784949127.606214:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289676032 00000100:00000001:0.0:1784949127.606222:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00010000:00000001:3.0:1784949127.606223:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949127.606229:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:3.0:1784949127.606236:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b77922d6 x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:33/0 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949127.606244:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949127.606268:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949127.606277:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949127.606278:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289675904:0@lo:400: 00000100:00001000:3.0:1784949127.606284:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=144 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949127.606290:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949127.606293:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949127.606296:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949127.606304:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000040:0.0:1784949127.606305:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289675904/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949127.606313:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949127.606323:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949127.606329:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949127.606337:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784949127.606338:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949127.606347:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1784949127.606349:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784949127.606354:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000229221cc. 00000400:00000010:3.0:1784949127.606356:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 02000000:00000001:0.0:1784949127.606362:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:3.0:1784949127.606366:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289676032, offset 224 00000020:00000001:0.0:1784949127.606367:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949127.606372:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949127.606378:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:3.0:1784949127.606380:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784949127.606383:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005a1c05a3. 02000000:00000001:0.0:1784949127.606391:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949127.606393:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:3.0:1784949127.606400:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000010:0.0:1784949127.606401:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000400:00000200:3.0:1784949127.606416:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3644:3644:240:4294967295] 192.168.202.137@tcp LPNI seq info [2341:2341:8:4294967295] 00000100:00000001:0.0:1784949127.606421:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949127.606426:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949127.606434:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949127.606445:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949127.606447:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949127.606455:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949127.606461:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949127.606465:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:3.0:1784949127.606469:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000100:00000001:0.0:1784949127.606474:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949127.606482:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949127.606492:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949127.606497:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949127.606509:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949127.606513:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949127.606517:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949127.606520:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949127.606524:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949127.606529:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b77922d6 x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:33/0 lens 224/224 e 0 to 0 dl 1784949138 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949127.606544:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b77922d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289676032:12345-192.168.202.137@tcp:400: Request processed in 549us (1044us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949127.606553:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 758 00000100:00000040:3.0:1784949127.606557:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949127.606559:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949127.606562:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949127.606567:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b0a11306. 00000020:00000010:3.0:1784949127.606572:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949127.606576:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000467d4d37. 00000020:00000040:3.0:1784949127.606580:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949127.606582:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949127.606658:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949127.606659:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949127.606667:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000800:00000001:3.0:1784949127.606671:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949127.606674:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949127.606679:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949127.606685:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949127.606687:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949127.606698:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000400:00000200:3.0:1784949127.606699:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:0.0:1784949127.606706:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00000001:0.0:1784949127.606714:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949127.606718:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949127.606721:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643b900 00000400:00000200:3.0:1784949127.606734:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8f41 [1] + 224 00000800:00000001:3.0:1784949127.606741:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949127.606756:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949127.606759:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949127.606764:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949127.606770:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949127.606774:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949127.606785:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949127.606807:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949127.606813:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949127.606817:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949127.606964:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949127.606969:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949127.606976:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949127.606978:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949127.606981:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001712b5b2. 00000400:00000200:2.0:1784949127.606988:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:2.0:1784949127.606991:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000100:00000001:2.0:1784949127.606995:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949127.607002:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949127.607015:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949127.607025:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949127.607027:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949127.607030:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949127.607032:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949127.607034:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949127.607036:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949127.607039:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949127.607045:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=144 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949127.607049:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=144 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949127.607052:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949127.607054:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949127.607056:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949127.607060:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949127.607079:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949127.607083:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949127.607085:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949127.607088:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949127.607089:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949127.607093:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949127.607100:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949127.607103:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949127.607105:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949127.607108:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949127.607117:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289676032:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949127.607121:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949127.607124:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949127.607126:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289676032/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949127.607135:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949127.607137:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949127.607140:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006b8bfa56. 02000000:00000001:2.0:1784949127.607148:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949127.607150:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949127.607151:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949127.607154:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949127.607156:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002a753d9d. 02000000:00000001:2.0:1784949127.607159:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949127.607160:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949127.607163:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949127.607168:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949127.607170:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949127.607172:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949127.607178:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949127.607179:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949127.607182:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949127.607184:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949127.648016:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306066987, at 4306066988 00000100:00000010:2.0:1784949127.648030:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002b896c5a. 00000100:00000040:0.0:1784949127.648033:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306066987, at 4306066988 00000100:00000010:0.0:1784949127.648048:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000100:00000040:3.0:1784949127.648059:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306066987, at 4306066988 00000100:00000040:1.0:1784949127.648061:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306066987, at 4306066988 00000100:00000010:3.0:1784949127.648077:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:1.0:1784949127.648082:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00010000:00000001:2.0:1784949128.436029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949128.436038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949128.436046:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949128.436048:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949128.436177:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949128.436184:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949128.436188:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949128.628057:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949128.628071:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949128.628073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949128.628078:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949128.628081:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949128.628086:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949128.628090:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949129.460036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949129.460044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949129.460052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949129.460054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949129.460177:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949129.460184:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949129.460189:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949129.652036:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949129.652051:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949129.652054:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949129.652058:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949129.652062:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949129.652066:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949129.652068:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949130.484023:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949130.484033:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949130.484041:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949130.484043:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949130.484140:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949130.484147:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949130.484152:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949130.676114:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949130.676129:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949130.676132:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949130.676138:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949130.676142:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949130.676147:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949130.676151:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949131.508044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949131.508053:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949131.508060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949131.508065:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949131.508068:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949131.508072:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949131.508077:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949131.508079:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949131.508082:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949131.508084:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949131.508189:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949131.508198:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949131.508204:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949131.700079:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949131.700096:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949131.700100:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949131.700107:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949131.700112:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949131.700118:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949131.700122:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949132.532054:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949132.532067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949132.532079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949132.532084:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949132.532202:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949132.532209:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949132.532213:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949132.724032:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949132.724035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949132.724045:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949132.724045:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000010:0.0:1784949132.724057:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949132.724066:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00100000:2.0:1784949132.724067:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949107/real 1784949107] req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949132 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949132.724076:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949132.724080:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949132.724088:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949132.724099:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005a1c05a3. 00000400:00000001:2.0:1784949132.724100:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949132.724118:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949132.724119:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949132.724123:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000681ff0f0. 00000100:00000040:0.0:1784949132.724128:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949132.724137:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000100:00000001:0.0:1784949132.724142:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1784949132.724145:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:0.0:1784949132.724146:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949132.724158:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_03+3] req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949132.724159:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949132.724181:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949132 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949132.724200:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949132 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949132.724208:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949132.724214:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:2.0:1784949132.724216:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1784949132.724220:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:2.0:1784949132.724224:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949132.724225:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949132.724234:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949132.724235:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:0.0:1784949132.724237:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000033c07e20. 00000100:00000001:2.0:1784949132.724241:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949132.724241:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784949132.724243:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949132.724250:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1784949132.724250:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 02000000:00000001:0.0:1784949132.724254:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1784949132.724257:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949132 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949132.724258:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949132.724259:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949132.724269:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000820f02b3. 00000100:00000001:3.0:1784949132.724273:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949132.724277:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.724279:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.724289:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949132.724289:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949132.724292:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949132.724293:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949132.724294:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949132.724296:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.724300:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949132.724307:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949132.724313:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1784949132.724313:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949132.724314:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949132.724319:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.724326:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949132.724332:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949132.724333:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033c07e20] to pc [ptlrpcd_00_00+0] req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949132.724339:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949132.724354:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1784949132.724356:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949132 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949132.724360:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949132.724370:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949132.724375:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949132.724382:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949132.724391:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289676160:0@lo:400: 00000100:00000001:0.0:1784949132.724393:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.724393:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1784949132.724402:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949132.724405:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949132.724406:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11409) 02000000:00000001:3.0:1784949132.724410:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949132.724415:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949132.724417:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949132.724417:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.724423:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.724424:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949132.724426:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949132.724428:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949132.724434:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000010:3.0:1784949132.724437:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000009713a832. 00000100:00000040:1.0:1784949132.724439:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949132.724441:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1784949132.724447:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949132.724451:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1784949132.724454:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1784949132.724458:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949132.724462:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001db3b9dd. 00000100:00000001:1.0:1784949132.724473:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949132.724476:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949132.724479:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 02000000:00000001:1.0:1784949132.724480:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949132.724484:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1784949132.724486:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00080000:0.0:1784949132.724487:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 02000000:00000001:1.0:1784949132.724493:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949132.724495:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289676160, portal 25 00000100:00000001:2.0:1784949132.724498:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1784949132.724500:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784949132.724504:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1784949132.724505:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000033c07e20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289676288:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949132.724507:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1784949132.724509:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000100:00080000:0.0:1784949132.724513:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1784949132.724517:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949132 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949132.724518:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949132.724522:0:89766:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1784949132.724524:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784949132.724525:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949132.724529:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949132.724534:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11379 02000000:00000001:1.0:1784949132.724538:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949132.724545:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006841dfc4. 00000100:00000001:3.0:1784949132.724546:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949132.724551:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949132.724557:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949132.724562:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019017dd3. 00000400:00000010:3.0:1784949132.724566:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000400:00000010:1.0:1784949132.724573:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000200:3.0:1784949132.724574:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289676160, offset 0 00000100:00100000:2.0:1784949132.724581:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289674880:192.168.202.137@tcp:8: 00000100:00000200:1.0:1784949132.724583:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289676288, portal 10 00000100:00000001:1.0:1784949132.724592:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1784949132.724592:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949132.724597:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554967168 : -115544154584448 : ffff96e9c7b5ca80) 00000100:00000040:1.0:1784949132.724611:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949132.724616:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000400:00000200:3.0:1784949132.724617:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00080000:0.0:1784949132.724629:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11378 00000100:00000001:1.0:1784949132.724638:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949132.724639:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643b980 00000400:00000200:0.0:1784949132.724644:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000040:1.0:1784949132.724651:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949132.724658:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000400:00000010:1.0:1784949132.724658:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000200:3.0:1784949132.724659:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8559 [2] + 0 00000100:00000001:0.0:1784949132.724668:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1784949132.724669:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ccad1f73. 00000100:00000040:0.0:1784949132.724676:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000400:00000200:3.0:1784949132.724677:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000085f9af2c 00000100:00000001:0.0:1784949132.724680:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000200:1.0:1784949132.724684:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289676288, offset 0 00000020:00000001:0.0:1784949132.724686:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:3.0:1784949132.724689:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:0.0:1784949132.724690:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000200:1.0:1784949132.724698:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949132.724699:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1784949132.724705:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000040:0.0:1784949132.724706:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:0.0:1784949132.724711:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784949132.724714:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:1.0:1784949132.724718:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:0.0:1784949132.724723:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949132.724727:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784949132.724729:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1784949132.724733:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949132.724737:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:1.0:1784949132.724741:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000040:0.0:1784949132.724743:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000200:1.0:1784949132.724748:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1784949132.724753:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000200:1.0:1784949132.724758:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00080000:0.0:1784949132.724763:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000200:1.0:1784949132.724770:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3645:3645:240:4294967295] 192.168.202.137@tcp LPNI seq info [2342:2342:8:4294967295] 00000100:00000001:0.0:1784949132.724772:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.724775:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949132.724779:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1784949132.724786:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949132.724788:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949132.724795:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949132.724799:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:2.0:1784949132.724801:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006d868603 x1871651289674880/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949132 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949132.724804:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949132.724805:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000010:0.0:1784949132.724814:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949132.724822:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=4 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:3.0:1784949132.724826:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000085f9af2c 02000000:00000001:0.0:1784949132.724831:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.0:1784949132.724833:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949132.724833:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000085f9af2c. 02000000:00000001:0.0:1784949132.724837:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.724839:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784949132.724844:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784949132.724847:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:1.0:1784949132.724848:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e706e181. 02000000:00000001:2.0:1784949132.724848:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000200:3.0:1784949132.724849:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:0.0:1784949132.724876:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:2.0:1784949132.724877:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009925ca9f. 02000000:00000001:0.0:1784949132.724880:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949132.724888:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000017ac11c7. 00000800:00000200:1.0:1784949132.724890:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:2.0:1784949132.724890:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949132.724893:0:337121:0:(events.c:337:request_in_callback()) incoming req@000000000086d093 x1871651289676160 msgsize 224 00000100:00000001:0.0:1784949132.724895:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949132.724897:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00100000:3.0:1784949132.724933:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784949132.724937:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000020:00000040:2.0:1784949132.724938:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00100000:0.0:1784949132.724940:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1784949132.724948:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:2.0:1784949132.724949:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784949132.724952:0:337121:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:2.0:1784949132.724956:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000d02b883a. 00000800:00000200:1.0:1784949132.724958:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e706e181 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1784949132.724966:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 02000000:00000001:2.0:1784949132.724970:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:3.0:1784949132.724973:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:0.0:1784949132.724974:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784949132.724976:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1784949132.724980:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.724981:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.724982:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:2.0:1784949132.724983:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006d868603. 00000100:00000040:0.0:1784949132.724988:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_rcv+-1] req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949132.724992:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2186 usecs (3 jiffies) 00000100:00000200:3.0:1784949132.724993:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949132.725002:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306072068 00000100:00000001:2.0:1784949132.725009:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000010:1.0:1784949132.725015:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:2.0:1784949132.725016:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949132.725028:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.725030:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784949132.725041:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.725042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949132.725043:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949132.725045:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306072068 00000100:00000001:2.0:1784949132.725051:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949132.725052:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11409) 00000100:00000010:1.0:1784949132.725054:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000040:3.0:1784949132.725055:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949132.725063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949132.725065:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949132.725071:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949132.725074:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.725075:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306072068 00000100:00000001:2.0:1784949132.725079:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949132.725080:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:1.0:1784949132.725082:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:1.0:1784949132.725089:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949132.725091:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949132.725093:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949132.725094:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306072068 00000100:00000001:3.0:1784949132.725097:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306072107 00000100:00000010:1.0:1784949132.725101:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:1.0:1784949132.725109:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784949132.725111:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306072068 00000100:00000001:1.0:1784949132.725119:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949132.725124:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306072107 00000100:00000001:1.0:1784949132.725126:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.725135:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949132.725136:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.725137:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306072068 00000100:00000001:2.0:1784949132.725142:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.725143:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949132.725145:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306072107 00000100:00000001:1.0:1784949132.725155:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1784949132.725155:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949132.725156:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306072068 00000100:00000001:1.0:1784949132.725161:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 02000000:00000001:2.0:1784949132.725163:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949132.725168:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949132.725169:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306072107 02000000:00000001:2.0:1784949132.725171:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.725172:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949132.725177:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.725183:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.725184:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306072068 00000100:00100000:2.0:1784949132.725187:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289676416:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784949132.725199:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1784949132.725200:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949132.725200:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949132.725208:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949132.725209:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e706e181. 00000100:00000001:3.0:1784949132.725210:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949132.725216:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949132.725218:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949132.725219:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.725220:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949132.725230:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949132.725231:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949132.725232:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.725232:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949132.725235:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000800:00000001:1.0:1784949132.725240:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:0.0:1784949132.725244:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:2.0:1784949132.725244:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949132.725247:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949132.725254:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949132.725258:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:0.0:1784949132.725264:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949132.725265:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949132.725266:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949132.725277:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949132.725284:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949132.725284:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949132.725285:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ba00 00000100:00000001:3.0:1784949132.725288:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949132.725293:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949132.725296:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949132.725296:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289676160 00000100:00000001:2.0:1784949132.725300:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949132.725304:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:1.0:1784949132.725306:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 171056 00000100:00000001:3.0:1784949132.725311:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949132.725315:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949132.725323:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949132.725325:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949132.725334:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949132.725335:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949132.725336:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289676160 00000400:00000200:1.0:1784949132.725342:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949132.725344:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784949132.725345:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949132.725351:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000200:1.0:1784949132.725352:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784949132.725356:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949132.725360:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949132.725362:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:3.0:1784949132.725364:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000200:1.0:1784949132.725368:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:3.0:1784949132.725370:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784949132.725377:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001725b510. 00000100:00000001:3.0:1784949132.725378:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949132.725384:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000001725b510 x1871651289676288 msgsize 224 00000020:00000010:3.0:1784949132.725386:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00100000:1.0:1784949132.725394:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784949132.725395:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949132.725402:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000020:00000010:3.0:1784949132.725410:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a5bcc72e. 00000020:00000010:3.0:1784949132.725415:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b0a11306. 00000100:00000001:1.0:1784949132.725417:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000010:3.0:1784949132.725422:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000800:00000001:1.0:1784949132.725426:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949132.725433:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949132.725439:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949132.725446:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949132.725450:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949132.725454:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949132.725465:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.725481:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949132.725487:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289676288 02000000:00000001:2.0:1784949132.725490:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949132.725494:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949132.725494:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949132.725501:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949132.725509:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949132.725514:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289676288 00000100:00000001:3.0:1784949132.725514:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000001:2.0:1784949132.725518:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949132.725519:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000040:2.0:1784949132.725524:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00100000:3.0:1784949132.725529:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 783 00000020:00000001:2.0:1784949132.725532:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:3.0:1784949132.725535:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000020:00000040:2.0:1784949132.725539:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:3.0:1784949132.725540:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782152784 : -115538927398832 : ffff96eaff465050) 00000020:00000001:2.0:1784949132.725550:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:3.0:1784949132.725552:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000086d093 x1871651289676160/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:38/0 lens 224/0 e 0 to 0 dl 1784949143 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949132.725560:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949132.725568:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949132.725572:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949132.725574:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000001:3.0:1784949132.725579:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949132.725581:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000020:00000010:2.0:1784949132.725583:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00100000:3.0:1784949132.725588:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000086d093 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289676160:12345-0@lo:400: 00000100:00000001:2.0:1784949132.725591:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784949132.725598:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289676160 00000020:00000010:2.0:1784949132.725602:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000020:00000001:3.0:1784949132.725605:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949132.725610:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949132.725613:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949132.725616:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006b8bfa56. 00000020:00000001:3.0:1784949132.725616:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949132.725621:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000010:2.0:1784949132.725625:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000067a9a1b7. 00000020:00000001:3.0:1784949132.725626:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949132.725629:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949132.725635:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000010:2.0:1784949132.725637:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000020:00000001:3.0:1784949132.725638:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949132.725641:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949132.725646:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949132.725649:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:2.0:1784949132.725653:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:3.0:1784949132.725654:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949132.725662:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000010:3.0:1784949132.725663:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000980961c7. 00000100:00000001:2.0:1784949132.725667:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784949132.725669:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.725675:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949132.725676:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949132.725686:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.725690:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00010000:00000040:3.0:1784949132.725691:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289676160 00000100:00000001:2.0:1784949132.725699:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00010000:00000001:3.0:1784949132.725699:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949132.725706:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:3.0:1784949132.725712:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000086d093 x1871651289676160/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:38/0 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949132.725728:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949132.725730:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949132.725731:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949132.725742:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=154 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949132.725748:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949132.725752:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949132.725753:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949132.725760:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949132.725760:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949132.725769:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00100000:2.0:1784949132.725773:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 759 02000000:00000001:3.0:1784949132.725774:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949132.725785:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949132.725786:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949132.725793:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949132.725794:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637813632 : -115544071737984 : ffff96e9cca5ed80) 00000400:00000010:3.0:1784949132.725798:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000100:00000200:3.0:1784949132.725805:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289676160, offset 224 00000100:00000040:2.0:1784949132.725810:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001725b510 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:38/0 lens 224/0 e 0 to 0 dl 1784949143 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949132.725815:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949132.725831:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949132.725838:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949132.725844:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643b980 00000020:00000010:2.0:1784949132.725847:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000400:00000200:3.0:1784949132.725874:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8f55 [1] + 224 00000100:00000001:2.0:1784949132.725877:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949132.725885:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000400:00000200:3.0:1784949132.725885:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949132.725894:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949132.725899:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949132.725900:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001725b510 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289676288:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949132.725934:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949132.725946:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949132.725963:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289676288 00000020:00000001:2.0:1784949132.725970:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949132.725974:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949132.725975:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1784949132.725978:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949132.725983:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000020:00000001:2.0:1784949132.725985:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:3.0:1784949132.725987:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000020:00000001:2.0:1784949132.725991:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:3.0:1784949132.725992:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949132.725996:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949132.726003:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949132.726004:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000040:3.0:1784949132.726011:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:2.0:1784949132.726013:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949132.726015:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949132.726021:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949132.726022:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949132.726024:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:2.0:1784949132.726030:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949132.726034:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949132.726034:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000086d093 x1871651289676160/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:38/0 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949132.726037:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.726041:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949132.726048:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000001:1.0:1784949132.726052:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1784949132.726056:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000086d093 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289676160:12345-0@lo:400: Request processed in 484us (1159us total) trans 0 rc 0/0 00000100:00000001:2.0:1784949132.726057:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000001:1.0:1784949132.726059:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1784949132.726064:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000010:1.0:1784949132.726066:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001db3b9dd. 02000000:00000010:2.0:1784949132.726073:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000004dc9167. 00000100:00100000:3.0:1784949132.726074:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 783 00000400:00000200:1.0:1784949132.726079:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 02000000:00000001:2.0:1784949132.726083:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949132.726083:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000010:1.0:1784949132.726086:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:3.0:1784949132.726091:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949132.726093:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.726096:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949132.726097:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:2.0:1784949132.726099:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949132.726106:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289676288 00000020:00000010:3.0:1784949132.726106:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b0a11306. 00000100:00000200:1.0:1784949132.726111:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949132.726116:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000010:3.0:1784949132.726116:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949132.726125:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 00010000:00000200:2.0:1784949132.726130:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001725b510 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:38/0 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949132.726134:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:1.0:1784949132.726143:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949132.726144:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784949132.726149:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000980961c7. 02000000:00000001:3.0:1784949132.726156:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949132.726160:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949132.726170:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:2.0:1784949132.726172:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949132.726179:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949132.726179:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000de368614. 00000100:00000001:1.0:1784949132.726181:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949132.726187:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949132.726189:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=149 v=5 (1 1 1 1) 00000400:00000010:3.0:1784949132.726190:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000047756a57. 00000100:00000001:1.0:1784949132.726195:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949132.726201:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949132.726203:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949132.726206:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949132.726212:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949132.726212:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949132.726219:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949132.726221:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949132.726228:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00001000:1.0:1784949132.726232:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=154 v=5 (5 5 5 5) 02000000:00000001:2.0:1784949132.726232:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949132.726240:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00001000:1.0:1784949132.726243:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=154 v=5 (1 1 1 1) 00000100:00000040:2.0:1784949132.726249:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949132.726254:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:2.0:1784949132.726258:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000100:00000001:1.0:1784949132.726259:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949132.726269:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:2.0:1784949132.726270:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289676288, offset 224 00010000:00000001:1.0:1784949132.726273:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.726280:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949132.726286:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949132.726292:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949132.726303:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949132.726316:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949132.726322:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3646:3646:240:4294967295] 192.168.202.137@tcp LPNI seq info [2343:2343:8:4294967295] 00000100:00000001:1.0:1784949132.726324:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949132.726328:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949132.726337:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949132.726357:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:1.0:1784949132.726366:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289676160:0@lo:400: 00000800:00000200:2.0:1784949132.726375:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949132.726379:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949132.726384:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e1f7df0e. 00000100:00000001:1.0:1784949132.726385:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949132.726391:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289676160/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949132.726399:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949132.726415:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000200:2.0:1784949132.726415:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000001:1.0:1784949132.726421:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:2.0:1784949132.726423:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e1f7df0e type 1, nob 320 niov 1 nkiov 1 02000000:00000010:1.0:1784949132.726427:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000009713a832. 02000000:00000001:1.0:1784949132.726435:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949132.726438:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949132.726440:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949132.726443:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949132.726445:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005a1c05a3. 00000100:00000001:2.0:1784949132.726446:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949132.726451:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1784949132.726455:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949132.726457:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949132.726462:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00010000:00000001:2.0:1784949132.726463:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949132.726468:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949132.726473:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949132.726478:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1784949132.726481:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001725b510 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:38/0 lens 224/224 e 0 to 0 dl 1784949143 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949132.726483:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.726492:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.726501:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949132.726505:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.726510:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949132.726510:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001725b510 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289676288:12345-192.168.202.137@tcp:400: Request processed in 666us (1119us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949132.726514:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949132.726538:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 759 00000100:00000040:2.0:1784949132.726543:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949132.726547:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949132.726550:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949132.726554:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000067a9a1b7. 00000020:00000010:2.0:1784949132.726560:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949132.726564:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006b8bfa56. 00000020:00000040:2.0:1784949132.726574:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949132.726578:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949132.726622:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949132.726628:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e1f7df0e. 00000400:00000200:3.0:1784949132.726634:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949132.726644:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949132.726650:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:3.0:1784949132.726653:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000100:00000001:3.0:1784949132.726660:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784949132.726661:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949132.726663:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949132.726674:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949132.726678:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949132.726681:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949132.726690:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949132.726701:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ba00 00000400:00000200:0.0:1784949132.726707:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8f5d [1] + 224 00000800:00000001:0.0:1784949132.726713:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949132.726721:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949132.726723:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949132.726727:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949132.726731:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949132.726735:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949132.726745:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949132.726765:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949132.726771:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949132.726773:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.726807:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949132.726812:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949132.726817:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949132.726820:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949132.726823:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019017dd3. 00000400:00000200:1.0:1784949132.726829:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:1.0:1784949132.726831:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000100:00000001:1.0:1784949132.726835:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949132.726841:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949132.726870:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949132.726881:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949132.726900:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.726902:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949132.726904:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949132.726905:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949132.726906:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949132.726908:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949132.726913:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=149 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949132.726915:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=149 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949132.726918:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949132.726920:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949132.726940:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949132.726943:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949132.726950:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.726953:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949132.726954:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949132.726956:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949132.726957:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949132.726959:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949132.726964:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949132.726966:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949132.726967:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949132.726970:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949132.726977:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000033c07e20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289676288:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949132.726980:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.726982:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949132.726983:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000033c07e20 x1871651289676288/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949148 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949132.726989:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949132.726990:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949132.726991:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006841dfc4. 02000000:00000001:1.0:1784949132.726994:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949132.726996:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949132.726997:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949132.726999:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949132.727000:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000820f02b3. 02000000:00000001:1.0:1784949132.727002:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949132.727003:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949132.727005:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000033c07e20. 00000100:00000001:1.0:1784949132.727012:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949132.727013:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.727015:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949132.727018:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949132.727019:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949132.727021:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949132.727022:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949132.767997:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306072107, at 4306072108 00000100:00000040:0.0:1784949132.768004:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306072107, at 4306072108 00000100:00000010:2.0:1784949132.768011:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:0.0:1784949132.768018:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:1.0:1784949132.768035:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306072107, at 4306072108 00000100:00000040:3.0:1784949132.768036:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306072107, at 4306072108 00000100:00000010:3.0:1784949132.768051:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:1.0:1784949132.768057:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949133.556051:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949133.556060:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949133.556068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949133.556070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.556238:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949133.556245:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949133.556250:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949133.748060:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949133.748071:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949133.748092:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949132/real 0] req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949133 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.748119:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.748123:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949133.748127:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949133.748133:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949133 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.748143:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.748154:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949133.748156:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.748159:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.748163:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949133.748166:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949133.748170:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949133 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.748180:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949133.748184:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.748186:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949133.748187:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.748191:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949133.748193:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949133.748197:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949133.748226:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:2.0:1784949133.748230:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949133.748234:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949133 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949133.748243:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289676416:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949133.748250:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.748253:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949133.748257:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289676416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949133 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.748267:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1784949133.748272:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949133.748275:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784949133.748280:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949133.748285:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 0000000017ac11c7. 02000000:00000001:2.0:1784949133.748296:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949133.748299:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949133.748302:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000040:0.0:1784949133.748305:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949133.748316:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949133.748319:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949133.748321:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.748326:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.748330:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949133.748331:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949133.748335:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000617daa1a. 00000100:00000001:2.0:1784949133.748335:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.748337:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949133.748350:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949133.748360:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949133.748366:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949133.748375:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949133.748388:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949133.748399:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949133.748409:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949133.748431:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949133.748435:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949133.748444:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000617daa1a] to pc [ptlrpcd_00_01+1] req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949133.748503:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949133.748509:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949133.748517:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949133.748523:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949133.748527:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784949133.748532:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949133.748535:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1784949133.748539:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949133.748545:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1784949133.748547:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11379 00000100:00000040:2.0:1784949133.748555:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949133.748569:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784949133.748576:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949133.748580:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949133.748583:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11404 02000000:00000001:2.0:1784949133.748588:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949133.748594:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949133.748596:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 02000000:00000001:2.0:1784949133.748600:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949133.748607:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1784949133.748608:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289676544:0@lo:400: 00000100:00000001:1.0:1784949133.748615:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.748621:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949133.748623:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949133.748625:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949133.748633:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949133.748636:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949133.748638:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949133.748645:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784949133.748646:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949133.748654:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000010:2.0:1784949133.748656:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006b8bfa56. 02000000:00000001:2.0:1784949133.748665:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949133.748667:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949133.748672:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000010:2.0:1784949133.748674:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001712b5b2. 00000020:00000001:0.0:1784949133.748678:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949133.748688:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:2.0:1784949133.748688:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000020:00000001:0.0:1784949133.748699:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000200:2.0:1784949133.748703:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289676544, portal 25 00000020:00000001:0.0:1784949133.748706:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:2.0:1784949133.748711:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949133.748717:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554966272 : -115544154585344 : ffff96e9c7b5c700) 00000100:00000040:0.0:1784949133.748721:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000040:2.0:1784949133.748729:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949133.748730:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949133.748741:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949133.748750:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784949133.748759:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1784949133.748762:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000040:2.0:1784949133.748771:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949133.748776:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000100:00080000:0.0:1784949133.748778:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000200:2.0:1784949133.748781:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289676544, offset 0 00000100:00080000:0.0:1784949133.748794:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000400:00000200:2.0:1784949133.748796:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949133.748808:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949133.748815:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1784949133.748821:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:0.0:1784949133.748827:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949133.748836:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949133.748876:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1784949133.748881:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643bb00 00000100:00000001:0.0:1784949133.748883:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949133.748899:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000400:00000200:2.0:1784949133.748900:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x85e9 [2] + 0 00000400:00000010:2.0:1784949133.748941:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f13fb872. 00000400:00000200:2.0:1784949133.748950:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000002e1eb6f6 00000400:00000200:2.0:1784949133.748962:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:0.0:1784949133.748965:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:2.0:1784949133.748974:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784949133.748980:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1784949133.748983:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e1eb6f6 02000000:00000001:0.0:1784949133.748986:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949133.748989:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000002e1eb6f6. 00000100:00000001:0.0:1784949133.748998:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949133.749000:0:337120:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:0.0:1784949133.749008:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1784949133.749008:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949133.749018:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000ffe216dc x1871651289676544 msgsize 224 00000100:00100000:2.0:1784949133.749023:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949133.749026:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 02000000:00000001:0.0:1784949133.749032:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.749043:0:337120:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:0.0:1784949133.749046:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001f4abcc4. 00000400:00000200:2.0:1784949133.749056:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000100:00000001:0.0:1784949133.749057:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949133.749063:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949133.749071:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1784949133.749078:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1784949133.749082:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949133.749095:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949133.749102:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289676544 02000000:00000001:1.0:1784949133.749109:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949133.749116:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949133.749122:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949133.749133:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949133.749147:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1784949133.749147:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289676544 00000100:00000001:2.0:1784949133.749148:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949133.749156:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.749157:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:2.0:1784949133.749159:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949133.749163:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949133.749171:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784949133.749178:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_rcv+-1] req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949133.749178:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949133.749186:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.749190:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949133.749197:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784949133.749203:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949133.749207:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.749216:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949133.749220:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306073131 00000020:00000010:1.0:1784949133.749228:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec77eb13. 00000100:00000010:2.0:1784949133.749232:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 00000100:00000001:1.0:1784949133.749236:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949133.749243:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec77eb13. 00000100:00000001:0.0:1784949133.749249:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.749251:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:1.0:1784949133.749257:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006841dfc4. 00000100:00000040:0.0:1784949133.749262:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:1.0:1784949133.749267:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000169a7180. 00000100:00000001:2.0:1784949133.749269:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306073131 00000100:00000001:0.0:1784949133.749279:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000020:00000010:1.0:1784949133.749280:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000010:2.0:1784949133.749284:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003b6d02be. 00000100:00000010:0.0:1784949133.749292:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949133.749297:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1784949133.749301:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000020:00000040:0.0:1784949133.749304:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949133.749309:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949133.749311:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306073131 00000100:00000001:0.0:1784949133.749315:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784949133.749315:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949133.749319:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949133.749325:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949133.749326:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949133.749328:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1784949133.749329:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e36e3817. 02000000:00000001:0.0:1784949133.749336:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949133.749337:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949133.749338:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.749338:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949133.749350:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306073131 00000100:00000040:3.0:1784949133.749351:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949133.749356:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000057c7ab4. 00000100:00000010:2.0:1784949133.749361:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000094130822. 00000100:00000001:1.0:1784949133.749363:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949133.749373:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.749374:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949133.749388:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949133.749388:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949133.749390:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949133.749392:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.749394:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949133.749398:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949133.749398:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949133.749402:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949133.749402:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949133.749410:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949133.749410:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1784949133.749415:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 784 02000000:00000001:3.0:1784949133.749422:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.749424:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949133.749426:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949133.749430:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949133.749435:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782151760 : -115538927399856 : ffff96eaff464c50) 00000100:00100000:3.0:1784949133.749439:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289676672:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784949133.749441:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.749449:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784949133.749454:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ffe216dc x1871651289676544/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:39/0 lens 224/0 e 0 to 0 dl 1784949144 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.749456:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949133.749458:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949133.749465:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949133.749468:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949133.749473:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949133.749474:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949133.749484:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949133.749486:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949133.749496:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000010:3.0:1784949133.749499:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000000239f355. 00000020:00000010:1.0:1784949133.749504:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec77eb13. 00000100:00000040:0.0:1784949133.749508:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_02+2] req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949133.749510:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.749511:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949133.749523:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec77eb13. 00000400:00000010:3.0:1784949133.749527:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fed0cd52. 00000100:00100000:1.0:1784949133.749540:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ffe216dc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289676544:12345-0@lo:400: 00000400:00000010:3.0:1784949133.749544:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000100:00000200:3.0:1784949133.749559:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289676672, portal 4 00000100:00000200:1.0:1784949133.749561:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289676544 00000020:00000001:1.0:1784949133.749569:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949133.749570:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1784949133.749577:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949133.749577:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 00000020:00000001:1.0:1784949133.749583:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.749590:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949133.749593:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000040:3.0:1784949133.749600:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949133.749601:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949133.749606:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949133.749614:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.749615:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949133.749621:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949133.749629:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949133.749632:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11410) 00000020:00000001:1.0:1784949133.749638:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949133.749648:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949133.749648:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949133.749650:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.749659:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784949133.749660:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:3.0:1784949133.749661:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949133.749666:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.749667:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1784949133.749670:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e5630c1a. 00000400:00000010:3.0:1784949133.749672:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bb55de7f. 00000100:00000001:0.0:1784949133.749677:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949133.749680:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949133.749684:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949133.749684:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289676672, offset 0 00000100:00000001:1.0:1784949133.749691:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.749698:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949133.749701:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00010000:00000040:1.0:1784949133.749708:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289676544 00000100:00000001:2.0:1784949133.749716:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1784949133.749717:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949133.749724:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949133.749724:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00010000:00000200:1.0:1784949133.749730:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ffe216dc x1871651289676544/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:39/0 lens 224/224 e 0 to 0 dl 1784949144 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949133.749733:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784949133.749737:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1784949133.749741:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949133.749751:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:2.0:1784949133.749752:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949133.749763:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00100000:2.0:1784949133.749765:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289676800:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784949133.749765:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1784949133.749769:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949133.749776:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:2.0:1784949133.749779:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00001000:1.0:1784949133.749780:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=5 v=5 (1 1 1 1) 02000000:00000001:2.0:1784949133.749787:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:3.0:1784949133.749790:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3647:3647:240:4294967295] 192.168.202.137@tcp LPNI seq info [2344:2344:8:4294967295] 02000000:00000001:2.0:1784949133.749794:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.749795:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949133.749800:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784949133.749805:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 02000000:00000010:2.0:1784949133.749812:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000008fae13bf. 00000100:00000001:1.0:1784949133.749816:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784949133.749819:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949133.749821:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:1.0:1784949133.749831:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:2.0:1784949133.749834:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000800:00000200:3.0:1784949133.749839:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949133.749841:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949133.749870:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000100:00000001:1.0:1784949133.749873:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000010:3.0:1784949133.749875:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000200:2.0:1784949133.749882:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289676800, portal 10 00000100:00000040:1.0:1784949133.749888:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949133.749891:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000800:00000200:3.0:1784949133.749891:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949133.749896:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000400:00000010:1.0:1784949133.749900:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000040:2.0:1784949133.749942:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949133.749943:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949133.749950:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 616 niov 1 nkiov 1 00000100:00000200:1.0:1784949133.749958:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289676544, offset 224 00000100:00000001:2.0:1784949133.749962:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949133.749973:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949133.749979:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:2.0:1784949133.749982:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000100:00000001:3.0:1784949133.749982:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949133.749992:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289676800, offset 0 00000100:00000040:3.0:1784949133.749992:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000400:00000200:1.0:1784949133.749996:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949133.750002:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306073132 00000400:00000200:2.0:1784949133.750008:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949133.750016:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4306073131 00000400:00000200:1.0:1784949133.750018:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643bb00 00000400:00000200:2.0:1784949133.750025:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949133.750031:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306073132 00000400:00000200:2.0:1784949133.750037:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949133.750038:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8f71 [1] + 224 00000100:00000001:3.0:1784949133.750043:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4306073131 00000400:00000200:2.0:1784949133.750048:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949133.750054:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949133.750056:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784949133.750058:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306073132 00000100:00000001:3.0:1784949133.750068:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4306073131 00000400:00000200:1.0:1784949133.750069:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784949133.750070:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949133.750076:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306073132 00000400:00000200:2.0:1784949133.750083:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3648:3648:239:4294967295] 192.168.202.137@tcp LPNI seq info [2345:2345:7:4294967295] 00000100:00000001:3.0:1784949133.750083:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4306073131 00000100:00000001:3.0:1784949133.750088:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949133.750092:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949133.750097:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949133.750101:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949133.750106:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949133.750109:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949133.750112:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:2.0:1784949133.750117:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949133.750119:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949133.750119:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:2.0:1784949133.750125:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000de6364cf. 00000100:00000001:3.0:1784949133.750125:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:1.0:1784949133.750132:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949133.750132:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949133.750143:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949133.750143:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949133.750162:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:1.0:1784949133.750165:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949133.750174:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000de6364cf type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949133.750199:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000200:0.0:1784949133.750211:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949133.750211:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949133.750212:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000800:00000001:3.0:1784949133.750216:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784949133.750219:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000800:00000010:0.0:1784949133.750226:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000001:1.0:1784949133.750228:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:2.0:1784949133.750228:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2187 usecs (3 jiffies) 00000100:00000001:1.0:1784949133.750234:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949133.750239:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.750242:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306073093 00000400:00000200:0.0:1784949133.750245:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949133.750246:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949133.750247:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949133.750248:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306073132 00000800:00000001:3.0:1784949133.750258:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.750261:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306073093 00000100:00000040:1.0:1784949133.750262:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784949133.750273:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306073132 00010000:00000001:1.0:1784949133.750274:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:0.0:1784949133.750276:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949133.750282:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:1.0:1784949133.750283:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.750285:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306073093 00000020:00000001:1.0:1784949133.750294:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784949133.750295:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306073132 00000400:00000200:0.0:1784949133.750296:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bb55de7f 00000100:00000040:1.0:1784949133.750305:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ffe216dc x1871651289676544/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:39/0 lens 224/224 e 0 to 0 dl 1784949144 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949133.750307:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bb55de7f. 00000100:00000001:2.0:1784949133.750308:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306073093 00000400:00000200:3.0:1784949133.750312:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643bb80 00000100:00000001:2.0:1784949133.750319:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306073132 00000100:00000001:0.0:1784949133.750325:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949133.750333:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000100:3.0:1784949133.750338:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289676672 offset 0 length 520: 4 00000100:00000200:0.0:1784949133.750342:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.750347:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949133.750356:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 000000001cbb52f7 is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:2.0:1784949133.750358:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949133.750362:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ffe216dc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289676544:12345-0@lo:400: Request processed in 859us (1335us total) trans 0 rc 0/0 00000400:00000200:3.0:1784949133.750370:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:2.0:1784949133.750371:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.750384:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949133.750391:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000001:3.0:1784949133.750392:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949133.750396:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1784949133.750399:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 784 00000100:00000001:2.0:1784949133.750402:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:3.0:1784949133.750407:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949133.750408:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000040:0.0:1784949133.750409:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949158 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.750411:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949133.750418:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.750419:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949133.750426:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784949133.750433:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949133.750437:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000169a7180. 00000100:00000001:0.0:1784949133.750440:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949133.750447:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1784949133.750450:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949133.750457:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006841dfc4. 00000800:00000001:0.0:1784949133.750463:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949133.750466:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1784949133.750467:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784949133.750475:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784949133.750478:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949133.750483:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000e5630c1a. 00000400:00000001:2.0:1784949133.750488:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1784949133.750490:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000400:00000001:2.0:1784949133.750497:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784949133.750498:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949133.750507:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001712b5b2. 00000100:00000200:1.0:1784949133.750510:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000010:1.0:1784949133.750521:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019017dd3. 00000400:00000200:2.0:1784949133.750523:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:2.0:1784949133.750531:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000800:00000001:0.0:1784949133.750535:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949133.750539:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000010:1.0:1784949133.750541:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d00a8655. 00000800:00000001:0.0:1784949133.750548:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949133.750554:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949133.750556:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949133.750563:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949133.750566:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949133.750571:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000de6364cf. 00000400:00000200:3.0:1784949133.750582:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949133.750587:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784949133.750589:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949133.750591:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949133.750602:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949133.750611:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643bc00 00000400:00000200:3.0:1784949133.750616:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:3.0:1784949133.750625:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000100:00000001:2.0:1784949133.750626:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949133.750629:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949133.750632:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949133.750633:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949133.750638:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000200:3.0:1784949133.750639:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.750645:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:0.0:1784949133.750646:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 171280 00000100:00000001:2.0:1784949133.750650:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949133.750661:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949133.750669:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:2.0:1784949133.750671:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=5 v=5 (5 5 5 5) 00000100:00000001:3.0:1784949133.750672:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00001000:2.0:1784949133.750678:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=5 v=5 (1 1 1 1) 00000100:00000040:3.0:1784949133.750682:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.750685:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949133.750688:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949133.750693:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949133.750694:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784949133.750695:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949133.750702:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949133.750703:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949133.750705:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949133.750709:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949133.750720:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:2.0:1784949133.750727:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949133.750730:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949133.750736:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949133.750745:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000033c07e20. 00000100:00000040:0.0:1784949133.750753:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000033c07e20 x1871651289676800 msgsize 224 00000100:00000001:2.0:1784949133.750764:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784949133.750770:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784949133.750777:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949133.750784:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949133.750790:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949133.750796:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00100000:2.0:1784949133.750807:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289676544:0@lo:400: 00000800:00000001:0.0:1784949133.750813:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949133.750815:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.750821:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:0.0:1784949133.750824:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949133.750826:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000617daa1a x1871651289676544/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.750874:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949133.750878:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949133.750882:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006b8bfa56. 00000100:00000001:1.0:1784949133.750892:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:2.0:1784949133.750895:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1784949133.750900:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289676800 00000020:00000001:2.0:1784949133.750901:0:337120:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:1.0:1784949133.750934:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949133.750937:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:2.0:1784949133.750941:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949133.750943:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949133.750952:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:1.0:1784949133.750954:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949133.750956:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 00000100:00100000:1.0:1784949133.750962:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289676800 02000000:00000001:2.0:1784949133.750964:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:1.0:1784949133.750968:0:342763:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1784949133.750969:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:1.0:1784949133.750975:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000010:2.0:1784949133.750976:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000617daa1a. 00000020:00000001:1.0:1784949133.750982:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949133.750987:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949133.750992:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949133.750992:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949133.750999:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949133.751001:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949133.751005:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949133.751008:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949133.751013:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1784949133.751015:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec77eb13. 00000100:00000001:2.0:1784949133.751019:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.751022:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949133.751025:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1784949133.751028:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec77eb13. 00000100:00000001:2.0:1784949133.751031:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949133.751038:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006841dfc4. 00000020:00000010:1.0:1784949133.751045:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000169a7180. 00000020:00000010:1.0:1784949133.751049:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949133.751058:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949133.751061:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949133.751063:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949133.751065:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949133.751071:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949133.751074:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949133.751076:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.751093:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949133.751103:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949133.751105:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949133.751111:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 760 00000100:00000040:1.0:1784949133.751114:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949133.751116:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554967168 : -115544154584448 : ffff96e9c7b5ca80) 00000100:00000040:1.0:1784949133.751121:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000033c07e20 x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:39/0 lens 224/0 e 0 to 0 dl 1784949144 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949133.751144:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949133.751147:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec77eb13. 00000100:00000001:1.0:1784949133.751150:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949133.751153:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec77eb13. 00000100:00100000:1.0:1784949133.751158:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000033c07e20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289676800:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949133.751168:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289676800 00000020:00000001:1.0:1784949133.751170:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949133.751174:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949133.751176:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.751179:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949133.751181:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949133.751184:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949133.751187:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949133.751190:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949133.751191:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949133.751194:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.751196:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949133.751200:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949133.751202:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949133.751205:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e5630c1a. 02000000:00000001:1.0:1784949133.751207:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.751210:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.751212:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949133.751214:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289676800 00010000:00000001:1.0:1784949133.751217:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949133.751221:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000033c07e20 x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:39/0 lens 224/224 e 0 to 0 dl 1784949144 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949133.751230:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949133.751233:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949133.751235:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=150 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949133.751240:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949133.751242:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949133.751245:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949133.751248:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949133.751250:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949133.751252:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949133.751255:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949133.751258:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000200:1.0:1784949133.751262:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289676800, offset 224 00000400:00000200:1.0:1784949133.751268:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949133.751277:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949133.751283:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3649:3649:240:4294967295] 192.168.202.137@tcp LPNI seq info [2346:2346:8:4294967295] 00000400:00000200:1.0:1784949133.751295:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949133.751301:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949133.751306:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000600b3b47. 00000800:00000200:1.0:1784949133.751311:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949133.751318:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949133.751322:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000600b3b47 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949133.751338:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949133.751342:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949133.751345:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949133.751346:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949133.751349:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949133.751353:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000033c07e20 x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:39/0 lens 224/224 e 0 to 0 dl 1784949144 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949133.751364:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000033c07e20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289676800:12345-192.168.202.137@tcp:400: Request processed in 219us (609us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949133.751372:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 760 00000100:00000040:1.0:1784949133.751375:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949133.751378:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949133.751380:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949133.751383:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000169a7180. 00000020:00000010:1.0:1784949133.751387:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949133.751390:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006841dfc4. 00000020:00000040:1.0:1784949133.751394:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949133.751397:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949133.751511:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949133.751518:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000600b3b47. 00000400:00000200:0.0:1784949133.751554:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949133.751555:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949133.751564:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949133.751570:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949133.751573:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000800:00000001:3.0:1784949133.751580:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:0.0:1784949133.751581:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000800:00000001:3.0:1784949133.751591:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949133.751592:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949133.751599:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949133.751609:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949133.751624:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643bc00 00000400:00000200:3.0:1784949133.751634:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8f81 [1] + 224 00000800:00000001:3.0:1784949133.751643:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949133.751650:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949133.751654:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949133.751661:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949133.751666:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949133.751671:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949133.751682:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949133.751706:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949133.751716:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949133.751721:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.751774:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.751779:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949133.751785:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949133.751788:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949133.751791:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000400:00000200:2.0:1784949133.751799:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784949133.751802:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949133.751807:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949133.751815:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949133.751830:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.751875:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949133.751878:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.751883:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949133.751886:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949133.751888:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949133.751892:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949133.751896:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949133.751938:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=150 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949133.751947:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=150 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949133.751950:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949133.751952:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949133.751955:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949133.751961:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949133.751971:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.751974:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949133.751977:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949133.751980:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949133.751982:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949133.751986:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.751994:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949133.751997:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949133.751999:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949133.752002:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949133.752010:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289676800:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949133.752014:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.752017:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949133.752020:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289676800/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949133.752028:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949133.752030:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949133.752032:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000008fae13bf. 02000000:00000001:2.0:1784949133.752038:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949133.752039:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949133.752041:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949133.752044:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949133.752046:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000057c7ab4. 02000000:00000001:2.0:1784949133.752049:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949133.752050:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949133.752052:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949133.752057:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949133.752059:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.752061:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949133.752066:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949133.752068:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949133.752070:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949133.752072:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949133.793017:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306073132, at 4306073133 00000100:00000010:0.0:1784949133.793032:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000040:2.0:1784949133.793034:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306073132, at 4306073133 00000100:00000040:3.0:1784949133.793034:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306073132, at 4306073133 00000100:00000040:1.0:1784949133.793038:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306073132, at 4306073133 00000100:00000010:2.0:1784949133.793057:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e36e3817. 00000100:00000010:3.0:1784949133.793059:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000094130822. 00000100:00000010:1.0:1784949133.793064:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003b6d02be. 00010000:00000001:2.0:1784949134.580014:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949134.580024:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949134.580033:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949134.580035:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949134.580169:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949134.580177:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949134.580181:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949135.604022:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949135.604032:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949135.604041:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949135.604043:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949135.604161:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949135.604168:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949135.604173:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949136.628031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949136.628041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949136.628049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949136.628051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949136.628122:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949136.628129:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949136.628134:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949137.652034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949137.652043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949137.652051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949137.652053:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949137.652171:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949137.652177:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949137.652182:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949138.676044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949138.676054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949138.676061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949138.676063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949138.676204:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949138.676213:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949138.676220:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949138.868022:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949138.868037:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949138.868040:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949138.868045:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949138.868046:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949138.868053:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949138.868060:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784949138.868061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949138.868068:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949138.868072:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949138.868082:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949138.868089:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949138.868094:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949138.868101:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949138.868113:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000635133da. 00000100:00000001:0.0:1784949138.868124:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949138.868138:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949138.868154:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949138.868157:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949138.868164:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_03+3] req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949138.868201:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949138.868205:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949138.868210:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949138.868214:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949138.868219:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949138.868223:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949138.868226:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949138.868229:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949138.868230:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949138.868235:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949138.868238:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949138.868239:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.868246:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:0.0:1784949138.868248:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000008f5595bf. 00000100:00000001:0.0:1784949138.868256:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949138.868260:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949138.868269:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949138.868300:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949138.868308:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949138.868308:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.868314:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949138.868316:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949138.868319:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949138.868324:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949138.868325:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949138.868327:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_00+0] req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949138.868337:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949138.868352:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289676928:0@lo:400: 00000100:00000001:1.0:1784949138.868360:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949138.868368:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949138.868371:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949138.868375:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949138.868378:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949138.868384:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006841dfc4. 00000100:00000001:2.0:1784949138.868385:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949138.868388:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11415) 02000000:00000001:1.0:1784949138.868393:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.868394:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949138.868402:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:1.0:1784949138.868407:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d6b8e52f. 00000100:00000040:2.0:1784949138.868415:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949138.868421:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000200:1.0:1784949138.868436:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289676928, portal 25 00000100:00000001:2.0:1784949138.868436:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949138.868442:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.868446:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1784949138.868450:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949138.868451:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 02000000:00000001:2.0:1784949138.868457:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949138.868463:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949138.868466:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949138.868473:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289677056:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949138.868491:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949138.868498:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784949138.868499:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949138.868506:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949138.868507:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784949138.868515:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:1.0:1784949138.868516:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000200:1.0:1784949138.868523:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289676928, offset 0 02000000:00000010:2.0:1784949138.868525:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000008fae13bf. 02000000:00000001:2.0:1784949138.868532:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949138.868539:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:2.0:1784949138.868545:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000400:00000010:2.0:1784949138.868558:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000200:1.0:1784949138.868563:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:2.0:1784949138.868569:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677056, portal 10 00000100:00000001:2.0:1784949138.868575:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949138.868577:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000400:00000200:1.0:1784949138.868584:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643bc80 00000100:00000040:2.0:1784949138.868591:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949138.868603:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8605 [2] + 0 00000100:00000001:0.0:1784949138.868614:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784949138.868617:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001a3c4d34. 00000100:00000001:0.0:1784949138.868626:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.868627:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784949138.868629:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f7c52f62 00000100:00000040:2.0:1784949138.868639:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949138.868644:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1784949138.868646:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000100:00000200:2.0:1784949138.868656:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289677056, offset 0 00000400:00000200:1.0:1784949138.868659:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784949138.868669:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949138.868680:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f7c52f62 00000400:00000200:2.0:1784949138.868687:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000010:1.0:1784949138.868689:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f7c52f62. 00000400:00000200:2.0:1784949138.868700:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949138.868701:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949138.868706:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1784949138.868712:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:1.0:1784949138.868715:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000014ab6f86 x1871651289676928 msgsize 224 00000400:00000200:2.0:1784949138.868722:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949138.868734:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00100000:1.0:1784949138.868736:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949138.868742:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949138.868747:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3650:3650:240:4294967295] 192.168.202.137@tcp LPNI seq info [2347:2347:8:4294967295] 00000100:00000001:1.0:1784949138.868761:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949138.868773:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949138.868783:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949138.868788:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000800:00000010:2.0:1784949138.868791:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002f91e44d. 00000400:00000010:1.0:1784949138.868794:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:1.0:1784949138.868825:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949138.868832:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949138.868845:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949138.868853:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949138.868861:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949138.868872:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949138.868882:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949138.868886:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002f91e44d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949138.868888:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949138.868898:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.868945:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306078251 00000100:00000001:3.0:1784949138.868948:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000010:1.0:1784949138.868953:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00100000:3.0:1784949138.868956:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289676928 02000000:00000001:3.0:1784949138.868962:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949138.868966:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949138.868967:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949138.868970:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.868973:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949138.868979:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.868981:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306078251 00000100:00000040:2.0:1784949138.868984:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2188 usecs (3 jiffies) 00000100:00100000:3.0:1784949138.868990:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289676928 00000100:00000010:1.0:1784949138.868991:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:2.0:1784949138.868993:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306078212 00000020:00000001:3.0:1784949138.868997:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784949138.869002:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949138.869005:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306078251 00000100:00000001:2.0:1784949138.869013:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306078212 00000100:00000001:1.0:1784949138.869015:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306078251 00000100:00000001:2.0:1784949138.869017:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306078251 00000100:00000010:1.0:1784949138.869025:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:2.0:1784949138.869026:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306078212 00000100:00000001:1.0:1784949138.869035:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949138.869041:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306078251 00000100:00000010:1.0:1784949138.869044:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:2.0:1784949138.869047:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306078251 00000100:00000001:1.0:1784949138.869049:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949138.869059:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306078212 00000100:00000001:1.0:1784949138.869062:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949138.869066:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306078251 00000100:00000001:1.0:1784949138.869069:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.869075:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949138.869077:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.869085:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.869086:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949138.869099:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.869103:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949138.869104:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949138.869105:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949138.869110:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000010:0.0:1784949138.869116:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002f91e44d. 00000100:00000001:1.0:1784949138.869118:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949138.869119:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949138.869124:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949138.869127:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:0.0:1784949138.869131:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949138.869131:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.869135:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949138.869141:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.869144:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949138.869152:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949138.869153:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949138.869163:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000020:00000040:3.0:1784949138.869164:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784949138.869165:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949138.869171:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000020:00000001:3.0:1784949138.869175:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949138.869181:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1784949138.869183:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000200:0.0:1784949138.869192:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949138.869193:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000001:1.0:1784949138.869199:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949138.869206:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000001:1.0:1784949138.869218:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949138.869220:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949138.869223:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949138.869228:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949138.869235:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949138.869241:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949138.869251:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949138.869258:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784949138.869265:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949138.869281:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643bd00 00000020:00000010:3.0:1784949138.869290:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000400:00000200:1.0:1784949138.869292:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 171504 00000100:00000001:3.0:1784949138.869302:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:1.0:1784949138.869307:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784949138.869308:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000020:00000010:3.0:1784949138.869322:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a5bcc72e. 00000800:00000001:1.0:1784949138.869325:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949138.869329:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b0a11306. 00000400:00000200:1.0:1784949138.869331:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949138.869343:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949138.869344:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949138.869349:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949138.869352:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784949138.869359:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000021f0c077. 00000100:00000040:3.0:1784949138.869359:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000040:1.0:1784949138.869366:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000021f0c077 x1871651289677056 msgsize 224 00000100:00000001:3.0:1784949138.869368:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949138.869375:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949138.869380:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949138.869381:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784949138.869394:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.869407:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949138.869415:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949138.869417:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949138.869426:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949138.869437:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949138.869443:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949138.869453:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 785 00000100:00000001:0.0:1784949138.869459:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784949138.869459:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949138.869466:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782155856 : -115538927395760 : ffff96eaff465c50) 00000100:00100000:0.0:1784949138.869467:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677056 02000000:00000001:0.0:1784949138.869474:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949138.869478:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784949138.869483:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000014ab6f86 x1871651289676928/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:44/0 lens 224/0 e 0 to 0 dl 1784949149 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949138.869484:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949138.869493:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949138.869502:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677056 00000020:00000001:0.0:1784949138.869508:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949138.869512:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000040:0.0:1784949138.869513:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949138.869521:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000010:3.0:1784949138.869522:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000020:00000040:0.0:1784949138.869524:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:3.0:1784949138.869530:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949138.869531:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000010:3.0:1784949138.869540:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000020:00000001:0.0:1784949138.869541:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949138.869551:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:3.0:1784949138.869556:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000014ab6f86 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289676928:12345-0@lo:400: 00000020:00000010:0.0:1784949138.869557:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:0.0:1784949138.869565:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784949138.869567:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289676928 00000020:00000010:0.0:1784949138.869572:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000020:00000001:3.0:1784949138.869575:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000010:0.0:1784949138.869582:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000020:00000001:3.0:1784949138.869583:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949138.869589:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949138.869593:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000265803cd. 00000020:00000001:3.0:1784949138.869596:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000010:0.0:1784949138.869600:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000020:00000001:3.0:1784949138.869601:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949138.869609:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949138.869611:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949138.869614:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:3.0:1784949138.869616:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949138.869622:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784949138.869624:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000001:3.0:1784949138.869627:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949138.869630:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:3.0:1784949138.869633:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949138.869636:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949138.869640:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949138.869646:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000001:3.0:1784949138.869647:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949138.869654:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949138.869659:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949138.869659:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008618f1b2. 02000000:00000001:3.0:1784949138.869669:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949138.869675:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949138.869677:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949138.869681:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289676928 00010000:00000001:3.0:1784949138.869685:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949138.869686:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:3.0:1784949138.869693:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000014ab6f86 x1871651289676928/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:44/0 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949138.869706:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00010000:00000001:3.0:1784949138.869707:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949138.869713:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00010000:00000001:3.0:1784949138.869715:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.869721:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00001000:3.0:1784949138.869722:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=10 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949138.869729:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949138.869731:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 761 00000100:00000001:3.0:1784949138.869737:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949138.869740:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000040:3.0:1784949138.869746:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949138.869747:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949138.869748:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637817216 : -115544071734400 : ffff96e9cca5fb80) 00000100:00000001:3.0:1784949138.869758:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:2.0:1784949138.869759:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949138.869767:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000021f0c077 x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:44/0 lens 224/0 e 0 to 0 dl 1784949149 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949138.869771:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949138.869777:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949138.869786:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949138.869792:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:3.0:1784949138.869799:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:0.0:1784949138.869821:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000100:00000001:0.0:1784949138.869829:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949138.869831:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000400:00000010:3.0:1784949138.869834:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000100:00100000:0.0:1784949138.869840:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000021f0c077 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289677056:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949138.869848:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289676928, offset 224 00000100:00000200:0.0:1784949138.869849:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677056 00000020:00000001:0.0:1784949138.869855:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949138.869863:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949138.869866:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949138.869873:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:3.0:1784949138.869874:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:0.0:1784949138.869877:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949138.869884:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949138.869888:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949138.869895:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949138.869898:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784949138.869900:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:0.0:1784949138.869902:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949138.869952:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:3.0:1784949138.869958:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643bc80 00000100:00000001:0.0:1784949138.869962:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949138.869968:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:3.0:1784949138.869974:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8f95 [1] + 224 02000000:00000010:0.0:1784949138.869978:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009898daab. 02000000:00000001:0.0:1784949138.869984:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949138.869985:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949138.869989:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949138.869999:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949138.869999:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00010000:00000040:0.0:1784949138.870009:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677056 00000100:00000001:3.0:1784949138.870013:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:0.0:1784949138.870015:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000200:3.0:1784949138.870023:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:0.0:1784949138.870026:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000021f0c077 x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:44/0 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949138.870079:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:3.0:1784949138.870083:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949138.870085:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949138.870094:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=5 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949138.870107:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949138.870115:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949138.870119:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949138.870123:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949138.870125:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949138.870129:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949138.870132:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1784949138.870134:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949138.870142:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000400:00000200:3.0:1784949138.870147:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000010:3.0:1784949138.870153:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000100:00000001:3.0:1784949138.870158:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000200:0.0:1784949138.870160:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289677056, offset 224 00000100:00000001:3.0:1784949138.870162:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949138.870171:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949138.870176:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949138.870179:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949138.870183:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949138.870186:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1784949138.870190:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949138.870196:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949138.870199:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:0.0:1784949138.870200:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000020:00000001:3.0:1784949138.870202:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:1.0:1784949138.870207:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949138.870215:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d6b8e52f. 00000100:00000040:3.0:1784949138.870215:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000014ab6f86 x1871651289676928/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:44/0 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949138.870219:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3651:3651:240:4294967295] 192.168.202.137@tcp LPNI seq info [2348:2348:8:4294967295] 00000400:00000200:1.0:1784949138.870229:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949138.870236:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784949138.870246:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949138.870260:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949138.870271:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:3.0:1784949138.870272:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000014ab6f86 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289676928:12345-0@lo:400: Request processed in 750us (1546us total) trans 0 rc 0/0 00000800:00000200:0.0:1784949138.870308:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00100000:3.0:1784949138.870309:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 785 00000100:00000200:1.0:1784949138.870310:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1784949138.870321:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003a8a98bc. 00000100:00000040:3.0:1784949138.870323:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949138.870334:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949138.870339:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949138.870340:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000200:0.0:1784949138.870343:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949138.870348:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949138.870350:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b0a11306. 00000100:00000001:1.0:1784949138.870355:0:337122:0:(client.c:1580:after_reply()) Process entered 00000800:00000200:0.0:1784949138.870360:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000001:1.0:1784949138.870363:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:3.0:1784949138.870364:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000800:00000200:0.0:1784949138.870366:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003a8a98bc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949138.870369:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:3.0:1784949138.870375:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 00000100:00000001:1.0:1784949138.870377:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949138.870384:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949138.870386:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:1.0:1784949138.870391:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=10 v=5 (5 5 5 5) 02000000:00000001:3.0:1784949138.870395:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784949138.870396:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949138.870399:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=10 v=5 (1 1 1 1) 02000000:00000010:3.0:1784949138.870401:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008618f1b2. 00000100:00000040:0.0:1784949138.870410:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 02000000:00000001:3.0:1784949138.870414:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949138.870415:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00010000:00000001:0.0:1784949138.870417:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949138.870424:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949138.870425:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949138.870426:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949138.870433:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:0.0:1784949138.870438:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00010000:00000001:1.0:1784949138.870439:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949138.870441:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949138.870448:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949138.870450:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000021f0c077 x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:44/0 lens 224/224 e 0 to 0 dl 1784949149 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949138.870453:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b69fa69e. 00000100:00000040:1.0:1784949138.870463:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949138.870466:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e6a263f4. 00000100:00000001:1.0:1784949138.870502:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784949138.870504:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000021f0c077 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289677056:12345-192.168.202.137@tcp:400: Request processed in 708us (1132us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949138.870513:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949138.870519:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949138.870526:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 761 00000100:00000040:1.0:1784949138.870532:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949138.870535:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949138.870544:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949138.870551:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949138.870557:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000265803cd. 00000100:00100000:1.0:1784949138.870559:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289676928:0@lo:400: 00000020:00000010:0.0:1784949138.870568:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949138.870573:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949138.870577:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000100:00000001:1.0:1784949138.870578:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:2.0:1784949138.870582:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:1.0:1784949138.870589:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289676928/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1784949138.870592:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000020:00000040:0.0:1784949138.870593:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949138.870607:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949138.870609:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949138.870611:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:3.0:1784949138.870615:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784949138.870620:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949138.870626:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006841dfc4. 00000400:00000200:2.0:1784949138.870627:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949138.870630:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949138.870636:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000001:3.0:1784949138.870641:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949138.870642:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:2.0:1784949138.870644:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000020:00000040:1.0:1784949138.870649:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000800:00000001:3.0:1784949138.870649:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949138.870651:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000020:00000001:1.0:1784949138.870661:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:2.0:1784949138.870664:0:337112:0:(events.c:386:reply_out_callback()) Process entered 02000000:00000010:1.0:1784949138.870666:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000635133da. 00000100:00000001:2.0:1784949138.870672:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 02000000:00000001:1.0:1784949138.870677:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1784949138.870678:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:1.0:1784949138.870684:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949138.870688:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000400:00000200:3.0:1784949138.870698:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643bd00 00000100:00000001:1.0:1784949138.870702:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949138.870706:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949138.870712:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8f9d [1] + 224 00000100:00000001:1.0:1784949138.870713:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949138.870724:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000001:3.0:1784949138.870726:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949138.870728:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949138.870734:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:3.0:1784949138.870735:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949138.870737:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949138.870745:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949138.870752:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949138.870758:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949138.870766:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949138.870778:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949138.870820:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949138.870831:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949138.870837:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949138.870868:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949138.870875:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949138.870882:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949138.870885:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949138.870889:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000400:00000200:2.0:1784949138.870897:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784949138.870902:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949138.870949:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949138.870960:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949138.870974:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949138.870984:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949138.870986:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949138.870989:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949138.870992:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949138.870994:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949138.870996:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949138.870999:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949138.871006:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=5 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949138.871010:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=5 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949138.871014:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949138.871015:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949138.871018:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949138.871022:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949138.871032:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.871035:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949138.871037:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949138.871039:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949138.871041:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949138.871044:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949138.871052:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949138.871055:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949138.871057:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949138.871060:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949138.871069:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289677056:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949138.871075:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949138.871078:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949138.871080:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289677056/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949138.871088:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949138.871090:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949138.871092:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000008fae13bf. 02000000:00000001:2.0:1784949138.871098:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949138.871100:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949138.871101:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949138.871104:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949138.871106:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000008f5595bf. 02000000:00000001:2.0:1784949138.871109:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949138.871111:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949138.871112:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949138.871118:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949138.871120:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949138.871122:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949138.871127:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949138.871129:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949138.871131:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949138.871134:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949138.912014:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306078251, at 4306078252 00000100:00000040:2.0:1784949138.912024:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306078251, at 4306078252 00000100:00000010:3.0:1784949138.912029:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:1.0:1784949138.912043:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306078251, at 4306078252 00000100:00000040:0.0:1784949138.912047:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306078251, at 4306078252 00000100:00000010:2.0:1784949138.912047:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:1.0:1784949138.912060:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:0.0:1784949138.912068:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949139.700020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949139.700028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949139.700035:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949139.700037:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949139.700157:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949139.700164:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949139.700168:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949139.892039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949139.892053:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949139.892056:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949139.892062:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949139.892066:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949139.892071:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949139.892075:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949140.724056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949140.724067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949140.724077:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949140.724080:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949140.724189:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949140.724196:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949140.724200:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949140.916081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949140.916095:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949140.916099:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949140.916103:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949140.916107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949140.916111:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949140.916114:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949141.108023:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.11@tcp 00000800:00000010:2.0:1784949141.108030:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000062b4a223. 00000400:00000010:3.0:1784949141.108035:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000054872d18. 00000400:00000200:2.0:1784949141.108040:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949141.108055:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784949141.108055:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000061508776. 00000400:00000010:3.0:1784949141.108070:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bfef55e5. 00000400:00000200:2.0:1784949141.108076:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11414 health :900 00000400:00000200:2.0:1784949141.108085:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:3.0:1784949141.108090:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001f662712. 00000400:00000200:2.0:1784949141.108092:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000010:3.0:1784949141.108103:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000383d6bd1. 00000400:00000200:2.0:1784949141.108108:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:3.0:1784949141.108133:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003d75c981. 00000400:00000010:3.0:1784949141.108139:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a7ce911a. 00000400:00000010:3.0:1784949141.108149:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003c8b2437. 00000400:00000010:3.0:1784949141.108156:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000004d10c17. 00000400:00000010:3.0:1784949141.108161:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000096383a5. 00000400:00000010:3.0:1784949141.108164:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000756d087e. 00000400:00000010:3.0:1784949141.108167:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000dceddabe. 00000400:00000010:3.0:1784949141.108170:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000067f2d25d. 00000400:00000010:3.0:1784949141.108175:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000147b89d3. 00000400:00000010:3.0:1784949141.108178:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000038aa1d2e. 00000400:00000010:3.0:1784949141.108181:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009e875900. 00000400:00000010:3.0:1784949141.108184:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000005547c433. 00000400:00000010:3.0:1784949141.108186:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000735cb99e. 00000400:00000010:3.0:1784949141.108189:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c3236cd0. 00000400:00000010:3.0:1784949141.108191:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006b55cc75. 00000400:00000010:3.0:1784949141.108193:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000063d06960. 00000400:00000010:3.0:1784949141.108196:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000a5d96d2. 00000400:00000010:3.0:1784949141.108199:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000056670828. 00000400:00000010:3.0:1784949141.108202:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001fcbecf4. 00000400:00000010:3.0:1784949141.108205:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000da72cb2d. 00000400:00000010:3.0:1784949141.108208:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000038468d93. 00000400:00000010:3.0:1784949141.108216:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008c245eeb. 00000400:00000010:3.0:1784949141.108219:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000acbf6818. 00000400:00000010:3.0:1784949141.108224:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000451e3fe4. 00000400:00000010:3.0:1784949141.108228:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009234a5fd. 00000400:00000010:3.0:1784949141.108231:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c986c2b1. 00000400:00000010:3.0:1784949141.108233:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000511e5266. 00000400:00000010:3.0:1784949141.108239:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000189c3332. 00000400:00000010:3.0:1784949141.108243:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ae557b6e. 00000400:00000200:1.0:1784949141.108248:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:3.0:1784949141.108248:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000325e9e7. 00000400:00000010:3.0:1784949141.108254:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ca3adacb. 00000400:00000200:1.0:1784949141.108263:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000010:3.0:1784949141.108269:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 0000000071d4fc18. 00000400:00000001:1.0:1784949141.108275:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:02000040:3.0:1784949141.108282:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000400:00000001:1.0:1784949141.108283:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949141.108290:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f43fd4d9 00000400:00000010:1.0:1784949141.108296:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f43fd4d9. 00000400:00000200:1.0:1784949141.108304:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949141.108310:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:1.0:1784949141.108316:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000053ab16eb. 00000400:00000200:1.0:1784949141.108321:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:1.0:1784949141.108324:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:1.0:1784949141.108329:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000400:00000010:3.0:1784949141.111854:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 0000000071d4fc18. 00010000:00000001:2.0:1784949141.748015:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949141.748022:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949141.748029:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949141.748033:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949141.748036:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949141.748039:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949141.748043:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949141.748045:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949141.748047:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949141.748050:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949141.748175:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949141.748181:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949141.748186:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949141.940119:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949141.940133:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949141.940136:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949141.940142:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949141.940145:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949141.940150:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949141.940155:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949142.132069:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:2.0:1784949142.132079:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000400:00000010:3.0:1784949142.132081:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000002443425e. 00000800:00000010:2.0:1784949142.132088:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003ed79484. 00000400:00000010:3.0:1784949142.132094:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000200:2.0:1784949142.132099:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949142.132103:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000400:00000200:2.0:1784949142.132112:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784949142.132121:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ca3adacb. 00000400:00000200:2.0:1784949142.132124:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11415 health :800 00000400:00000200:3.0:1784949142.132134:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949142.132148:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949142.132157:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:3.0:1784949142.132168:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949142.132178:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949142.132183:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3652:3652:241:4294967295] 192.168.202.137@tcp LPNI seq info [2349:2349:8:4294967295] 00000400:00000200:3.0:1784949142.132192:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949142.132201:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949142.132209:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949142.132213:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000800:00000010:3.0:1784949142.132219:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000800:00000200:3.0:1784949142.132231:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949142.132243:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949142.132249:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949142.132293:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949142.132308:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:1.0:1784949142.132318:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949142.132320:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949142.132323:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:1.0:1784949142.132326:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000400:00000200:1.0:1784949142.132333:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949142.132338:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:1.0:1784949142.132343:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b105ef1a. 00000400:00000200:1.0:1784949142.132350:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:1.0:1784949142.132354:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:1.0:1784949142.132360:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00000800:00000200:0.0:1784949142.132509:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949142.132517:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:0.0:1784949142.132526:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949142.132535:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784949142.132541:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949142.132542:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949142.132551:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949142.132563:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949142.132569:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949142.132575:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949142.132588:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949142.132606:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949142.132619:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949142.132631:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949142.132637:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3653:3653:242:4294967295] 192.168.202.137@tcp LPNI seq info [2350:2350:8:4294967295] 00000400:00000200:2.0:1784949142.132648:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949142.132654:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949142.132660:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f95b4924. 00000800:00000200:2.0:1784949142.132666:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949142.132672:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949142.132677:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f95b4924 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949142.132824:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949142.132828:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949142.132836:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f95b4924. 00000800:00000001:3.0:1784949142.132837:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949142.132845:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949142.132846:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949142.132856:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949142.132857:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949142.132866:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949142.132874:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949142.132885:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949142.132934:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8fb1 00000800:00000001:3.0:1784949142.132942:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949142.132953:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949142.132956:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949142.132960:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949142.132966:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000010:3.0:1784949142.132969:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000400:00000200:3.0:1784949142.132973:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949142.132978:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000002443425e. 00000400:00000010:3.0:1784949142.132982:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:3.0:1784949142.132989:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949142.132992:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949142.772016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949142.772025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949142.772032:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949142.772034:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949142.772139:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949142.772146:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949142.772151:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949142.963998:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949142.964013:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949142.964016:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949142.964021:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949142.964024:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949142.964029:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949142.964031:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949143.795988:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949143.795995:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949143.796000:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949143.796001:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949143.796045:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949143.796049:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949143.796052:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949143.987960:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1784949143.987972:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949143.987972:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949143.987975:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949143.987980:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949143.987981:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784949143.987984:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1784949143.987988:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000617daa1a. 00000100:00000001:3.0:1784949143.987988:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.987993:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949143.987995:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949143.988001:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949143.988003:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949143.988008:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949143.988014:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003ed79484. 00000100:00000001:0.0:1784949143.988020:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.988027:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949143.988036:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949143.988038:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.988042:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000617daa1a] to pc [ptlrpcd_00_01+1] req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949143.988064:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949143.988066:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949143.988069:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949143.988071:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949143.988073:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949143.988075:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949143.988077:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949143.988078:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949143.988079:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949143.988080:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949143.988082:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000062b4a223. 00000100:00000001:0.0:1784949143.988083:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.988088:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949143.988093:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949143.988094:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.988096:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_02+2] req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949143.988109:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949143.988115:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:1.0:1784949143.988118:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949143.988121:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949143.988125:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949143.988126:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11421) 00000100:00000001:1.0:1784949143.988129:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949143.988134:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949143.988134:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949143.988135:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949143.988139:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949143.988140:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949143.988140:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988142:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949143.988144:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949143.988148:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949143.988151:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949143.988152:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949143.988152:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949143.988180:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949143.988182:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949143.988183:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988184:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949143.988186:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949143.988188:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949143.988189:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949143.988191:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949143.988194:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949143.988195:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949143.988199:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289677184:0@lo:400: 00000100:00100000:3.0:1784949143.988201:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289677312:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949143.988206:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949143.988207:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949143.988210:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949143.988211:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949143.988213:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949143.988214:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949143.988219:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1784949143.988219:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949143.988225:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006841dfc4. 02000000:00000010:3.0:1784949143.988225:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a5bcc72e. 02000000:00000001:1.0:1784949143.988229:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949143.988230:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949143.988235:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d6b8e52f. 00000400:00000010:3.0:1784949143.988236:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ea556a29. 00000400:00000010:1.0:1784949143.988244:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000400:00000010:3.0:1784949143.988244:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000100:00000200:1.0:1784949143.988254:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677184, portal 25 00000100:00000200:3.0:1784949143.988254:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677312, portal 10 00000100:00000001:1.0:1784949143.988260:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949143.988260:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949143.988261:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554966272 : -115544154585344 : ffff96e9c7b5c700) 00000100:00000001:3.0:1784949143.988263:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000100:00000040:1.0:1784949143.988267:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949143.988272:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949143.988277:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949143.988282:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949143.988286:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000001:3.0:1784949143.988288:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:1.0:1784949143.988289:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289677184, offset 0 00000100:00000040:3.0:1784949143.988294:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949143.988298:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784949143.988298:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000100:00000200:3.0:1784949143.988303:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289677312, offset 0 00000400:00000200:3.0:1784949143.988307:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949143.988312:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784949143.988318:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949143.988322:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949143.988326:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949143.988329:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949143.988331:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949143.988334:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3654:3654:242:4294967295] 192.168.202.137@tcp LPNI seq info [2351:2351:8:4294967295] 00000400:00000200:3.0:1784949143.988342:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949143.988346:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949143.988350:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000400:00000200:1.0:1784949143.988351:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643bd80 00000400:00000200:1.0:1784949143.988361:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x861d [2] + 0 00000800:00000200:3.0:1784949143.988363:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:1.0:1784949143.988366:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fde3a35c. 00000400:00000200:1.0:1784949143.988372:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000abb53582 00000800:00000200:3.0:1784949143.988373:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949143.988378:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1784949143.988381:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949143.988387:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949143.988391:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949143.988392:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abb53582 00000400:00000010:1.0:1784949143.988397:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000abb53582. 00000100:00000040:3.0:1784949143.988398:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2190 usecs (3 jiffies) 00000100:00000001:3.0:1784949143.988402:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306083331 00000100:00000001:1.0:1784949143.988404:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000010:3.0:1784949143.988407:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000200:1.0:1784949143.988408:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784949143.988414:0:337119:0:(events.c:337:request_in_callback()) incoming req@0000000000407c50 x1871651289677184 msgsize 224 00000100:00000001:3.0:1784949143.988415:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00100000:1.0:1784949143.988423:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784949143.988423:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306083331 00000100:00000040:1.0:1784949143.988428:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000010:3.0:1784949143.988429:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003e9ea21c. 00000100:00000001:3.0:1784949143.988432:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949143.988436:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306083331 00000100:00000010:3.0:1784949143.988439:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000100:00000001:3.0:1784949143.988441:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949143.988446:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306083331 00000100:00000010:3.0:1784949143.988448:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000083930ef2. 00000100:00000001:3.0:1784949143.988450:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784949143.988452:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949143.988453:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988455:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988457:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949143.988459:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949143.988461:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784949143.988462:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949143.988463:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949143.988464:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988466:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949143.988493:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949143.988498:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000100:00000001:1.0:1784949143.988498:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949143.988505:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949143.988511:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949143.988511:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000800:00000001:2.0:1784949143.988512:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784949143.988517:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:3.0:1784949143.988521:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949143.988522:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784949143.988523:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0e1f1a0 00000800:00000001:2.0:1784949143.988525:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949143.988525:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677184 00000400:00000010:0.0:1784949143.988527:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0e1f1a0. 00000800:00000001:2.0:1784949143.988530:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949143.988530:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000200:1.0:1784949143.988531:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949143.988534:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949143.988534:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000001:2.0:1784949143.988535:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949143.988537:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949143.988542:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949143.988544:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949143.988551:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00100000:3.0:1784949143.988552:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677184 00000020:00000001:3.0:1784949143.988557:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949143.988561:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784949143.988565:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949143.988566:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949143.988568:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643be00 00000020:00000040:3.0:1784949143.988570:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949143.988574:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949143.988576:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949143.988576:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:0.0:1784949143.988580:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949143.988582:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:2.0:1784949143.988583:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 171728 00000100:00000001:3.0:1784949143.988591:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949143.988596:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000800:00000001:2.0:1784949143.988598:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949143.988603:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949143.988605:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949143.988611:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000400:00000200:2.0:1784949143.988612:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949143.988615:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949143.988616:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.988618:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784949143.988620:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009713a832. 00000100:00000001:1.0:1784949143.988621:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784949143.988622:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949143.988628:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b0a11306. 00000100:00000001:1.0:1784949143.988632:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949143.988634:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949143.988637:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306083370 00000100:00000001:2.0:1784949143.988637:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949143.988640:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:3.0:1784949143.988642:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949143.988645:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306083331 00000100:00000010:2.0:1784949143.988646:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007f0cb9f0. 00000100:00000001:3.0:1784949143.988646:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949143.988650:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:2.0:1784949143.988651:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000007f0cb9f0 x1871651289677312 msgsize 224 00000100:00000001:3.0:1784949143.988653:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949143.988654:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306083370 00000100:00100000:2.0:1784949143.988660:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949143.988662:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306083331 00000100:00000001:3.0:1784949143.988662:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949143.988670:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306083370 00000100:00000001:1.0:1784949143.988675:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306083331 00000100:00000001:2.0:1784949143.988675:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949143.988676:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949143.988681:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306083370 00000800:00000001:2.0:1784949143.988683:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949143.988688:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949143.988689:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306083331 00000100:00000001:3.0:1784949143.988692:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949143.988695:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949143.988696:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784949143.988701:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 786 00000100:00000001:1.0:1784949143.988703:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949143.988705:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949143.988708:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949143.988708:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988708:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782148688 : -115538927402928 : ffff96eaff464050) 00000100:00100000:0.0:1784949143.988712:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677312 02000000:00000001:0.0:1784949143.988715:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949143.988716:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.988718:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784949143.988718:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000000407c50 x1871651289677184/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:49/0 lens 224/0 e 0 to 0 dl 1784949154 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949143.988722:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949143.988727:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988730:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:0.0:1784949143.988731:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677312 00000020:00000010:3.0:1784949143.988733:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000020:00000001:0.0:1784949143.988734:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949143.988737:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:3.0:1784949143.988738:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949143.988740:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000010:3.0:1784949143.988741:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000020:00000040:0.0:1784949143.988743:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949143.988748:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00100000:3.0:1784949143.988748:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000000407c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289677184:12345-0@lo:400: 00000020:00000001:0.0:1784949143.988753:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:3.0:1784949143.988755:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677184 00000100:00000001:0.0:1784949143.988757:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:3.0:1784949143.988771:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000010:0.0:1784949143.988774:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000020:00000001:3.0:1784949143.988775:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949143.988777:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949143.988778:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949143.988780:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000020:00000001:3.0:1784949143.988782:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949143.988785:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000010:0.0:1784949143.988786:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000020:00000001:3.0:1784949143.988788:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000010:0.0:1784949143.988791:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c9127b8a. 00000020:00000001:3.0:1784949143.988791:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949143.988797:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000020:00000001:3.0:1784949143.988797:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949143.988799:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949143.988802:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.988804:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000001:3.0:1784949143.988804:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949143.988806:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949143.988807:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949143.988811:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784949143.988811:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949143.988813:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784949143.988816:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008618f1b2. 00000100:00000001:0.0:1784949143.988819:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000001:3.0:1784949143.988820:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.988822:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949143.988823:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.988825:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949143.988826:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949143.988830:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677184 00010000:00000001:3.0:1784949143.988832:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949143.988836:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000000407c50 x1871651289677184/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:49/0 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949143.988837:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949143.988846:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949143.988847:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00010000:00000001:3.0:1784949143.988848:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.988851:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00001000:3.0:1784949143.988853:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=15 v=5 (1 1 1 1) 00000100:00100000:0.0:1784949143.988857:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 762 00000100:00000001:3.0:1784949143.988860:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949143.988861:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949143.988863:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562015872 : -115544147535744 : ffff96e9c8215880) 00000100:00000040:3.0:1784949143.988864:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949143.988868:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000040:0.0:1784949143.988870:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000007f0cb9f0 x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:49/0 lens 224/0 e 0 to 0 dl 1784949154 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949143.988872:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949143.988875:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949143.988880:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949143.988884:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:3.0:1784949143.988887:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:0.0:1784949143.988888:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:0.0:1784949143.988891:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784949143.988891:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000020:00000010:0.0:1784949143.988896:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000200:3.0:1784949143.988896:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289677184, offset 224 00000100:00100000:0.0:1784949143.988900:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000007f0cb9f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289677312:12345-192.168.202.137@tcp:400: 00000400:00000200:3.0:1784949143.988924:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949143.988933:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:0.0:1784949143.988934:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677312 00000020:00000001:0.0:1784949143.988937:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949143.988940:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949143.988943:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949143.988943:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643bd80 00000020:00000001:0.0:1784949143.988949:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949143.988951:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:3.0:1784949143.988952:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8fb5 [1] + 224 00000020:00000001:0.0:1784949143.988956:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:3.0:1784949143.988959:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949143.988960:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949143.988964:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949143.988966:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784949143.988967:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:0.0:1784949143.988969:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949143.988972:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949143.988972:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949143.988976:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000200:3.0:1784949143.988978:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949143.988979:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949143.988983:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000348be837. 02000000:00000001:0.0:1784949143.988987:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.988990:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949143.988992:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949143.988993:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949143.988997:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677312 00010000:00000001:0.0:1784949143.989002:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949143.989006:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000007f0cb9f0 x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:49/0 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949143.989012:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949143.989012:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000001:0.0:1784949143.989014:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949143.989017:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=10 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949143.989020:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1784949143.989020:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:3.0:1784949143.989024:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000100:00000040:0.0:1784949143.989025:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949143.989028:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949143.989029:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784949143.989031:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 02000000:00000001:0.0:1784949143.989033:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949143.989035:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.989037:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949143.989038:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.989042:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949143.989043:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000010:0.0:1784949143.989046:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00010000:00000001:3.0:1784949143.989047:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949143.989049:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949143.989050:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949143.989052:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289677312, offset 224 00000100:00000001:1.0:1784949143.989053:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949143.989054:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:0.0:1784949143.989058:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949143.989061:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000000407c50 x1871651289677184/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:49/0 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949143.989068:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:0.0:1784949143.989069:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000001:1.0:1784949143.989073:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1784949143.989077:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000000407c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289677184:12345-0@lo:400: Request processed in 345us (661us total) trans 0 rc 0/0 00000400:00000010:1.0:1784949143.989078:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d6b8e52f. 00000400:00000200:0.0:1784949143.989081:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3655:3655:242:4294967295] 192.168.202.137@tcp LPNI seq info [2352:2352:8:4294967295] 00000400:00000200:1.0:1784949143.989085:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949143.989091:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00100000:3.0:1784949143.989094:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 786 00000100:00000001:1.0:1784949143.989097:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1784949143.989099:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949143.989103:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:1.0:1784949143.989106:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949143.989106:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:0.0:1784949143.989109:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:3.0:1784949143.989111:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b0a11306. 00000020:00000010:3.0:1784949143.989118:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949143.989124:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009713a832. 00000800:00000200:0.0:1784949143.989128:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000040:3.0:1784949143.989130:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:1.0:1784949143.989134:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1784949143.989135:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008f5595bf. 02000000:00000001:3.0:1784949143.989135:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784949143.989138:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008618f1b2. 02000000:00000001:3.0:1784949143.989143:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949143.989146:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949143.989149:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:3.0:1784949143.989152:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949143.989159:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000010:3.0:1784949143.989159:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e8e1fecb. 00000800:00000200:0.0:1784949143.989164:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949143.989164:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949143.989166:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007c7d797b. 00000100:00000001:1.0:1784949143.989170:0:337119:0:(client.c:1580:after_reply()) Process entered 00000800:00000200:0.0:1784949143.989172:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f5595bf type 1, nob 320 niov 1 nkiov 1 02000000:00000001:1.0:1784949143.989175:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949143.989178:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949143.989180:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949143.989182:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949143.989184:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.989188:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949143.989191:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949143.989192:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949143.989192:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=15 v=5 (5 5 5 5) 00000020:00000001:0.0:1784949143.989197:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00001000:1.0:1784949143.989200:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=15 v=5 (1 1 1 1) 00000100:00000040:0.0:1784949143.989203:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007f0cb9f0 x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:49/0 lens 224/224 e 0 to 0 dl 1784949154 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949143.989207:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949143.989211:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949143.989216:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000007f0cb9f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289677312:12345-192.168.202.137@tcp:400: Request processed in 331us (563us total) trans 0 rc 0/0 00010000:00000001:1.0:1784949143.989217:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949143.989221:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949143.989226:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 762 00000100:00000001:1.0:1784949143.989227:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949143.989229:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949143.989232:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949143.989234:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:1.0:1784949143.989236:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949143.989238:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c9127b8a. 00000020:00000010:0.0:1784949143.989243:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000800:00000200:3.0:1784949143.989246:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1784949143.989249:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000800:00000010:3.0:1784949143.989250:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008f5595bf. 00000100:00000001:1.0:1784949143.989255:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949143.989256:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:0.0:1784949143.989258:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949143.989262:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949143.989263:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949143.989267:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949143.989267:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949143.989274:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000400:00000010:3.0:1784949143.989275:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000100:00000040:1.0:1784949143.989277:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949143.989279:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949143.989283:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949143.989288:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:1.0:1784949143.989291:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289677184:0@lo:400: 00000800:00000001:3.0:1784949143.989293:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949143.989298:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949143.989300:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949143.989301:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949143.989306:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784949143.989310:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:1.0:1784949143.989313:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000617daa1a x1871651289677184/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949143.989319:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643be00 00000400:00000200:3.0:1784949143.989328:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8fb9 [1] + 224 00000100:00000001:1.0:1784949143.989333:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:3.0:1784949143.989334:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784949143.989338:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949143.989342:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006841dfc4. 00000800:00000001:3.0:1784949143.989344:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949143.989346:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784949143.989348:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1784949143.989351:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784949143.989353:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784949143.989354:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1784949143.989358:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000200:3.0:1784949143.989359:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949143.989363:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949143.989366:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003ed79484. 02000000:00000001:1.0:1784949143.989372:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:3.0:1784949143.989374:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949143.989376:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949143.989379:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000617daa1a. 00000100:00000001:1.0:1784949143.989390:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949143.989393:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949143.989394:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949143.989398:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949143.989400:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949143.989403:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949143.989404:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949143.989407:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949143.989411:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949143.989413:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949143.989422:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949143.989426:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949143.989430:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949143.989432:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949143.989434:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ea556a29. 00000400:00000200:2.0:1784949143.989439:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000010:2.0:1784949143.989441:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000100:00000001:2.0:1784949143.989444:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949143.989449:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949143.989459:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949143.989468:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949143.989469:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949143.989471:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949143.989473:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949143.989475:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949143.989476:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949143.989478:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949143.989483:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=10 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949143.989486:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=10 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949143.989489:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949143.989490:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949143.989492:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949143.989495:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949143.989505:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949143.989507:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949143.989509:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949143.989511:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949143.989513:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949143.989516:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949143.989523:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949143.989526:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949143.989527:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949143.989530:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949143.989539:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289677312:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949143.989543:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949143.989545:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949143.989548:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289677312/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949159 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949143.989555:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949143.989557:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949143.989559:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a5bcc72e. 02000000:00000001:2.0:1784949143.989562:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949143.989563:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949143.989565:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949143.989567:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949143.989569:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000062b4a223. 02000000:00000001:2.0:1784949143.989571:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949143.989573:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949143.989574:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949143.989582:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949143.989583:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949143.989585:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949143.989589:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949143.989590:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949143.989592:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949143.989594:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949144.030963:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306083370, at 4306083371 00000100:00000040:1.0:1784949144.030963:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306083370, at 4306083371 00000100:00000040:2.0:1784949144.030969:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306083370, at 4306083371 00000100:00000040:3.0:1784949144.030974:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306083370, at 4306083371 00000100:00000010:1.0:1784949144.030975:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003e9ea21c. 00000100:00000010:0.0:1784949144.030977:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:2.0:1784949144.030979:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00000100:00000010:3.0:1784949144.030984:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000083930ef2. 00000800:00000100:2.0:1784949144.180032:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000800:00000010:2.0:1784949144.180039:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f64a2f6c. 00000400:00000200:2.0:1784949144.180047:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949144.180055:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949144.180062:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11417 health :900 00000400:00000200:2.0:1784949144.180068:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949144.180072:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:2.0:1784949144.180077:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000800:00000200:2.0:1784949144.180142:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:2.0:1784949144.180155:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000400:00000200:1.0:1784949144.180158:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:2.0:1784949144.180165:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000400:00000200:1.0:1784949144.180173:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:1.0:1784949144.180184:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949144.180187:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949144.180191:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00000400:00000010:1.0:1784949144.180195:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000400:00000200:1.0:1784949144.180202:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949144.180211:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:1.0:1784949144.180217:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001879354a. 00000400:00000200:1.0:1784949144.180226:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000200:1.0:1784949144.180232:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:1.0:1784949144.180239:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000800:00000200:3.0:1784949144.180373:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784949144.820000:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949144.820006:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949144.820011:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949144.820012:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949144.820062:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949144.820066:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949144.820068:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949145.012021:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949145.012032:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949145.012033:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949145.012036:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949145.012038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949145.012041:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949145.012043:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949145.204048:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949145.204059:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000002443425e. 00000800:00000100:2.0:1784949145.204066:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000400:00000010:3.0:1784949145.204072:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000800:00000010:2.0:1784949145.204075:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000086e4d21. 00000400:00000010:3.0:1784949145.204082:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000400:00000200:2.0:1784949145.204086:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949145.204096:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000325e9e7. 00000400:00000200:2.0:1784949145.204097:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949145.204113:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949145.204114:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11418 health :800 00000400:00000200:2.0:1784949145.204126:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949145.204132:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:3.0:1784949145.204142:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949145.204152:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949145.204158:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3656:3656:243:4294967295] 192.168.202.137@tcp LPNI seq info [2353:2353:8:4294967295] 00000400:00000200:3.0:1784949145.204168:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949145.204177:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949145.204185:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949145.204189:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000800:00000010:3.0:1784949145.204194:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000800:00000200:3.0:1784949145.204207:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949145.204219:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949145.204225:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949145.204259:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000100:2.0:1784949145.204260:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000800:00000010:2.0:1784949145.204270:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f8a82730. 00000400:00000200:1.0:1784949145.204277:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000200:2.0:1784949145.204279:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000001:1.0:1784949145.204292:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1784949145.204293:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000001:1.0:1784949145.204299:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949145.204305:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000400:00000010:1.0:1784949145.204309:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000400:00000200:1.0:1784949145.204317:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949145.204319:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949145.204328:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:1.0:1784949145.204333:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:1.0:1784949145.204342:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000307ddfde. 00000400:00000200:1.0:1784949145.204352:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:2.0:1784949145.204357:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:1.0:1784949145.204362:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:1.0:1784949145.204373:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000400:00000200:1.0:1784949145.204384:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:1.0:1784949145.204388:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949145.204390:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949145.204391:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553902b5 00000400:00000010:1.0:1784949145.204394:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000553902b5. 00000400:00000200:1.0:1784949145.204401:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949145.204406:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:1.0:1784949145.204410:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004041a48e. 00000400:00000200:1.0:1784949145.204415:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:1.0:1784949145.204418:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:1.0:1784949145.204421:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000800:00000200:0.0:1784949145.204467:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949145.204476:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000400:00000200:0.0:1784949145.204486:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949145.204495:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:3.0:1784949145.204501:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949145.204502:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949145.204511:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949145.204522:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949145.204529:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949145.204536:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949145.204549:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949145.204565:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949145.204578:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949145.204589:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949145.204595:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3657:3657:245:4294967295] 192.168.202.137@tcp LPNI seq info [2354:2354:8:4294967295] 00000400:00000200:3.0:1784949145.204606:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949145.204611:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949145.204616:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949145.204622:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949145.204627:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949145.204630:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784949145.204697:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949145.204798:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949145.204811:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949145.204815:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949145.204819:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949145.204828:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000200:3.0:1784949145.204830:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949145.204837:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:3.0:1784949145.204849:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949145.204850:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8fd1 00000800:00000001:0.0:1784949145.204863:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949145.204865:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949145.204877:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949145.204886:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949145.204893:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949145.204901:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949145.204950:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:0.0:1784949145.204954:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000400:00000200:0.0:1784949145.204958:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949145.204961:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000002443425e. 00000400:00000010:0.0:1784949145.204966:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:0.0:1784949145.204973:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949145.204977:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949145.844036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949145.844046:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949145.844054:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949145.844056:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949145.844189:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949145.844196:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949145.844201:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949146.036080:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949146.036098:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949146.036102:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949146.036111:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949146.036115:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949146.036121:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949146.036124:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949146.228056:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000800:00000010:2.0:1784949146.228065:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000eafb2cdc. 00000400:00000200:2.0:1784949146.228077:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949146.228084:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949146.228093:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11419 health :900 00000400:00000200:2.0:1784949146.228100:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949146.228105:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:2.0:1784949146.228111:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000800:00000100:2.0:1784949146.228142:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:2.0:1784949146.228145:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000353a5c12. 00000400:00000200:2.0:1784949146.228149:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949146.228155:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949146.228159:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949146.228164:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:2.0:1784949146.228167:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:1.0:1784949146.228191:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949146.228209:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:1.0:1784949146.228221:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949146.228225:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949146.228230:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdaa2431 00000400:00000010:1.0:1784949146.228235:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdaa2431. 00000400:00000200:1.0:1784949146.228243:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949146.228250:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:1.0:1784949146.228257:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000461737cb. 00000400:00000200:1.0:1784949146.228267:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:1.0:1784949146.228273:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:1.0:1784949146.228280:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:1.0:1784949146.228287:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:1.0:1784949146.228291:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949146.228294:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949146.228297:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000400:00000010:1.0:1784949146.228300:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000400:00000200:1.0:1784949146.228306:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949146.228311:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:1.0:1784949146.228316:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f0e4e38. 00000400:00000200:1.0:1784949146.228323:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:1.0:1784949146.228326:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:1.0:1784949146.228330:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00010000:00000001:2.0:1784949146.868030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949146.868040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949146.868048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949146.868050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949146.868177:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949146.868185:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949146.868189:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949147.060032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949147.060047:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949147.060050:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949147.060056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949147.060060:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949147.060064:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949147.060068:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949147.252044:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949147.252054:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000007631eb20. 00000400:00000010:3.0:1784949147.252068:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:3.0:1784949147.252075:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 00000400:00000010:3.0:1784949147.252085:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ae557b6e. 00000400:00000200:3.0:1784949147.252091:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949147.252110:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949147.252118:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949147.252124:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3658:3658:247:4294967295] 192.168.202.137@tcp LPNI seq info [2355:2355:8:4294967295] 00000400:00000200:3.0:1784949147.252133:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949147.252141:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949147.252148:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949147.252154:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949147.252162:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949147.252172:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949147.252178:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949147.252202:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11421 health :800 00000800:00000200:2.0:1784949147.252380:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949147.252388:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:2.0:1784949147.252398:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949147.252408:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949147.252417:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:3.0:1784949147.252417:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949147.252426:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949147.252443:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949147.252451:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949147.252463:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949147.252477:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949147.252499:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949147.252515:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949147.252530:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949147.252536:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3659:3659:247:4294967295] 192.168.202.137@tcp LPNI seq info [2356:2356:8:4294967295] 00000400:00000200:3.0:1784949147.252547:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949147.252553:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949147.252558:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949147.252564:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949147.252571:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949147.252575:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:3.0:1784949147.252693:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949147.252699:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000800:00000001:0.0:1784949147.252706:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949147.252706:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949147.252718:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949147.252721:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949147.252729:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949147.252770:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949147.252773:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949147.252778:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949147.252791:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949147.252809:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x8fd5 00000800:00000001:0.0:1784949147.252816:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949147.252826:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949147.252829:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949147.252833:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949147.252841:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096db69f7 00000400:00000010:0.0:1784949147.252845:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000096db69f7. 00000400:00000200:0.0:1784949147.252848:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949147.252852:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000007631eb20. 00000400:00000010:0.0:1784949147.252857:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:0.0:1784949147.252864:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949147.252867:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949147.892021:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949147.892030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949147.892038:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949147.892040:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949147.892170:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949147.892177:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949147.892181:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949148.084033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949148.084048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949148.084051:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949148.084056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949148.084060:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949148.084065:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949148.084069:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949148.276045:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000010:2.0:1784949148.276052:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d104522b. 00000400:00000200:2.0:1784949148.276063:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949148.276072:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949148.276082:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11421 health :900 00000400:00000200:2.0:1784949148.276090:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949148.276095:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:2.0:1784949148.276101:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000800:00000100:2.0:1784949148.276134:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000800:00000010:2.0:1784949148.276136:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000000d1fbdb0. 00000400:00000200:2.0:1784949148.276140:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949148.276146:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949148.276152:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949148.276156:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:2.0:1784949148.276160:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:1.0:1784949148.276170:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949148.276185:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:1.0:1784949148.276195:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949148.276199:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949148.276204:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:1.0:1784949148.276208:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000400:00000200:1.0:1784949148.276214:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949148.276220:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:1.0:1784949148.276225:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000be26f34f. 00000400:00000200:1.0:1784949148.276231:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:1.0:1784949148.276234:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:1.0:1784949148.276239:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:1.0:1784949148.276243:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:1.0:1784949148.276246:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949148.276247:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949148.276249:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:1.0:1784949148.276252:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000400:00000200:1.0:1784949148.276254:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949148.276257:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:1.0:1784949148.276260:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dc7a25c6. 00000400:00000200:1.0:1784949148.276265:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:1.0:1784949148.276267:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:1.0:1784949148.276270:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00010000:00000001:2.0:1784949148.916017:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949148.916028:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949148.916036:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949148.916038:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949148.916163:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949148.916170:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949148.916175:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949149.108058:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949149.108068:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949149.108078:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949149.108086:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949149.108094:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949149.108095:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949149.108103:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949149.108110:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1784949149.108112:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.108116:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949149.108122:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949149.108125:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000635133da. 00000100:00000001:2.0:1784949149.108127:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949149.108135:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949149.108140:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.108141:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949149.108160:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949149.108177:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949149.108180:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949149.108187:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_03+3] req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949149.108221:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949149.108224:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949149.108229:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949149.108233:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949149.108237:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949149.108242:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949149.108246:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949149.108249:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949149.108251:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949149.108253:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949149.108259:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003a8a98bc. 00000100:00000001:0.0:1784949149.108263:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949149.108265:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949149.108273:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949149.108276:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949149.108284:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949149.108285:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949149.108294:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949149.108295:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949149.108300:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949149.108303:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949149.108311:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_00+0] req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949149.108357:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949149.108362:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949149.108369:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949149.108373:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949149.108377:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949149.108382:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289677440:0@lo:400: 00000100:00000001:0.0:1784949149.108383:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949149.108396:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949149.108398:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11426) 02000000:00000001:1.0:1784949149.108407:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949149.108412:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949149.108413:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.108419:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784949149.108421:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949149.108424:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1784949149.108432:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006841dfc4. 00000100:00000040:2.0:1784949149.108439:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949149.108441:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949149.108441:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949149.108450:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949149.108459:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d6b8e52f. 00000100:00000001:2.0:1784949149.108472:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949149.108475:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949149.108478:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 02000000:00000001:2.0:1784949149.108480:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949149.108487:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949149.108491:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949149.108492:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677440, portal 25 00000100:00100000:2.0:1784949149.108496:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289677568:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949149.108499:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949149.108505:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000100:00000001:2.0:1784949149.108508:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949149.108516:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949149.108519:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949149.108520:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949149.108525:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949149.108535:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000008fae13bf. 00000100:00000001:1.0:1784949149.108538:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784949149.108542:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949149.108550:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949149.108556:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000400:00000010:1.0:1784949149.108557:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000200:1.0:1784949149.108567:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289677440, offset 0 00000400:00000010:2.0:1784949149.108572:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000200:1.0:1784949149.108580:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000200:2.0:1784949149.108593:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677568, portal 10 00000100:00000001:2.0:1784949149.108601:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949149.108605:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949149.108607:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000100:00000040:2.0:1784949149.108622:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949149.108629:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643be80 00000100:00000001:2.0:1784949149.108647:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784949149.108649:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8641 [2] + 0 00000100:00000040:2.0:1784949149.108660:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949149.108661:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000084252b45. 00000400:00000010:2.0:1784949149.108669:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000400:00000200:1.0:1784949149.108673:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000042334b2c 00000100:00000200:2.0:1784949149.108681:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289677568, offset 0 00000400:00000200:1.0:1784949149.108686:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949149.108699:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784949149.108702:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949149.108739:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000042334b2c 00000400:00000010:1.0:1784949149.108745:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000042334b2c. 00000400:00000200:2.0:1784949149.108747:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949149.108756:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949149.108761:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1784949149.108763:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1784949149.108767:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000032a4c2ab x1871651289677440 msgsize 224 00000400:00000200:2.0:1784949149.108778:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00100000:1.0:1784949149.108780:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949149.108787:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949149.108794:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.108803:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949149.108810:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949149.108812:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3660:3660:249:4294967295] 192.168.202.137@tcp LPNI seq info [2357:2357:8:4294967295] 00000400:00000200:2.0:1784949149.108834:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949149.108844:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949149.108849:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000800:00000010:2.0:1784949149.108853:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f95b4924. 00000400:00000010:1.0:1784949149.108855:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:3.0:1784949149.108862:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949149.108865:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1784949149.108868:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677440 00000800:00000200:2.0:1784949149.108873:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:3.0:1784949149.108876:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000200:1.0:1784949149.108878:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949149.108883:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949149.108891:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949149.108900:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 02000000:00000001:3.0:1784949149.108941:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949149.108950:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677440 00000020:00000001:3.0:1784949149.108953:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949149.108958:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000200:2.0:1784949149.108961:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f95b4924 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:3.0:1784949149.108965:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949149.108966:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1784949149.108973:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000040:1.0:1784949149.108978:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949149.108981:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949149.108981:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784949149.108994:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:2.0:1784949149.108996:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2190 usecs (3 jiffies) 00000100:00000001:3.0:1784949149.109004:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949149.109008:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306088452 00000020:00000010:3.0:1784949149.109015:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000010:2.0:1784949149.109024:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e36e3817. 00000100:00000001:1.0:1784949149.109026:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949149.109027:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949149.109039:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784949149.109040:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000100:00000001:2.0:1784949149.109042:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949149.109054:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949149.109056:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a5bcc72e. 00000100:00000001:2.0:1784949149.109058:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306088452 00000100:00000001:1.0:1784949149.109061:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306088491 00000020:00000010:3.0:1784949149.109070:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b0a11306. 00000100:00000010:2.0:1784949149.109071:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000fbfea82a. 00000100:00000001:1.0:1784949149.109076:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306088452 00000020:00000010:3.0:1784949149.109083:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:2.0:1784949149.109084:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949149.109087:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306088491 00000100:00000001:2.0:1784949149.109100:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306088452 00000100:00000040:3.0:1784949149.109105:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949149.109106:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306088452 00000100:00000010:2.0:1784949149.109109:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000addba683. 00000100:00000001:3.0:1784949149.109115:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949149.109121:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949149.109122:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306088491 00000100:00000001:2.0:1784949149.109123:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949149.109125:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.109138:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306088452 00000100:00000001:1.0:1784949149.109142:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306088452 00000100:00000001:3.0:1784949149.109143:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949149.109145:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003b6d02be. 00000100:00000001:2.0:1784949149.109160:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949149.109163:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306088491 00000100:00000001:2.0:1784949149.109170:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949149.109174:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949149.109176:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306088452 00000100:00000001:2.0:1784949149.109177:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.109192:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949149.109197:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949149.109197:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949149.109200:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949149.109206:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949149.109213:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949149.109214:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f95b4924. 00000100:00000001:2.0:1784949149.109215:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949149.109216:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949149.109223:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949149.109227:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.109230:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949149.109231:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:3.0:1784949149.109234:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 787 00000100:00000001:2.0:1784949149.109237:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949149.109239:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949149.109247:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949149.109248:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949149.109254:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1784949149.109255:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949149.109259:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782149712 : -115538927401904 : ffff96eaff464450) 00000100:00000001:2.0:1784949149.109260:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949149.109263:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:0.0:1784949149.109272:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000100:00000001:1.0:1784949149.109276:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949149.109281:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000032a4c2ab x1871651289677440/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:55/0 lens 224/0 e 0 to 0 dl 1784949160 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949149.109282:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000100:00000001:1.0:1784949149.109282:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949149.109291:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949149.109294:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949149.109304:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949149.109312:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949149.109323:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:2.0:1784949149.109330:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949149.109335:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1784949149.109336:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e8ecb15f. 00000100:00000040:0.0:1784949149.109346:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949149.109349:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949149.109350:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949149.109362:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784949149.109365:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e8ecb15f. 00000800:00000001:2.0:1784949149.109371:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949149.109386:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949149.109388:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000032a4c2ab pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289677440:12345-0@lo:400: 00000400:00000200:2.0:1784949149.109393:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949149.109395:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:3.0:1784949149.109408:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677440 00000020:00000001:3.0:1784949149.109415:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:2.0:1784949149.109422:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643bf00 00000020:00000001:3.0:1784949149.109424:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949149.109431:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949149.109437:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:2.0:1784949149.109438:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 171952 00000020:00000001:3.0:1784949149.109444:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000001:2.0:1784949149.109452:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949149.109454:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949149.109461:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949149.109466:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949149.109467:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:2.0:1784949149.109476:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784949149.109477:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:2.0:1784949149.109486:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784949149.109487:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.109494:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:3.0:1784949149.109496:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000200:2.0:1784949149.109499:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784949149.109504:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000010:2.0:1784949149.109507:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000017f9754b. 02000000:00000001:3.0:1784949149.109513:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784949149.109514:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000017f9754b x1871651289677568 msgsize 224 00000100:00100000:2.0:1784949149.109524:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000010:3.0:1784949149.109528:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000980961c7. 02000000:00000001:3.0:1784949149.109536:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949149.109541:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949149.109545:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.109549:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000040:3.0:1784949149.109551:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677440 00010000:00000001:3.0:1784949149.109560:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:2.0:1784949149.109562:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949149.109569:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:3.0:1784949149.109571:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000032a4c2ab x1871651289677440/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:55/0 lens 224/224 e 0 to 0 dl 1784949160 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949149.109589:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949149.109593:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949149.109598:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=21 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949149.109604:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949149.109606:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784949149.109611:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00100000:0.0:1784949149.109613:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677568 00000100:00000001:3.0:1784949149.109618:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949149.109620:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949149.109625:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:3.0:1784949149.109628:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784949149.109630:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949149.109635:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949149.109638:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949149.109642:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949149.109645:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677568 00000100:00000040:3.0:1784949149.109651:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:0.0:1784949149.109653:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949149.109659:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:3.0:1784949149.109659:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000020:00000001:0.0:1784949149.109668:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:3.0:1784949149.109671:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289677440, offset 224 00000020:00000040:0.0:1784949149.109674:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949149.109683:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784949149.109683:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:0.0:1784949149.109691:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949149.109695:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949149.109703:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784949149.109705:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:0.0:1784949149.109712:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949149.109738:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000400:00000200:3.0:1784949149.109747:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643be80 00000020:00000010:0.0:1784949149.109753:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000400:00000200:3.0:1784949149.109762:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x8fd9 [1] + 224 00000020:00000010:0.0:1784949149.109765:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c9127b8a. 00000020:00000010:0.0:1784949149.109773:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:3.0:1784949149.109774:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949149.109784:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:0.0:1784949149.109787:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949149.109790:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949149.109794:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949149.109799:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:3.0:1784949149.109801:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949149.109804:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949149.109818:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949149.109825:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000040:3.0:1784949149.109829:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949149.109830:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949149.109856:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949149.109863:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949149.109872:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784949149.109875:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000100:00000001:0.0:1784949149.109877:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:3.0:1784949149.109881:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000100:00000001:3.0:1784949149.109888:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00100000:0.0:1784949149.109891:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 763 00000100:00000001:3.0:1784949149.109893:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000040:0.0:1784949149.109901:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949149.109943:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562023936 : -115544147527680 : ffff96e9c8217800) 00000100:00000001:3.0:1784949149.109945:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949149.109955:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000040:0.0:1784949149.109956:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000017f9754b x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:55/0 lens 224/0 e 0 to 0 dl 1784949160 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949149.109966:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949149.109973:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949149.109977:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949149.109983:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949149.109987:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949149.109988:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:3.0:1784949149.109995:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000032a4c2ab x1871651289677440/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:55/0 lens 224/224 e 0 to 0 dl 1784949160 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949149.109997:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000400:00000001:1.0:1784949149.109999:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949149.110010:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949149.110011:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784949149.110019:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d6b8e52f. 00000020:00000010:0.0:1784949149.110021:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00100000:3.0:1784949149.110028:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000032a4c2ab pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289677440:12345-0@lo:400: Request processed in 697us (1252us total) trans 0 rc 0/0 00000400:00000200:1.0:1784949149.110035:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000100:00100000:0.0:1784949149.110043:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000017f9754b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289677568:12345-192.168.202.137@tcp:400: 00000400:00000010:1.0:1784949149.110044:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00100000:3.0:1784949149.110052:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 787 00000100:00000001:1.0:1784949149.110058:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1784949149.110066:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:0.0:1784949149.110067:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677568 00000100:00000200:1.0:1784949149.110074:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949149.110077:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949149.110078:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:0.0:1784949149.110087:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949149.110087:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:0.0:1784949149.110095:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949149.110097:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b0a11306. 00000020:00000001:0.0:1784949149.110104:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000200:1.0:1784949149.110107:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949149.110109:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000010:3.0:1784949149.110111:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000001:0.0:1784949149.110124:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000010:3.0:1784949149.110125:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a5bcc72e. 00000020:00000001:0.0:1784949149.110133:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949149.110136:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:3.0:1784949149.110138:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:0.0:1784949149.110140:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784949149.110143:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949149.110146:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:3.0:1784949149.110150:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949149.110151:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:0.0:1784949149.110152:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949149.110158:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000980961c7. 00000020:00000001:0.0:1784949149.110160:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 02000000:00000001:1.0:1784949149.110161:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:3.0:1784949149.110168:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949149.110169:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949149.110171:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949149.110174:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949149.110177:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949149.110178:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000200:3.0:1784949149.110182:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:1.0:1784949149.110184:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949149.110186:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008f2ed6fd. 00000400:00000010:3.0:1784949149.110193:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f6dbc39d. 02000000:00000001:0.0:1784949149.110195:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949149.110197:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=21 v=5 (5 5 5 5) 00000100:00000001:0.0:1784949149.110206:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949149.110207:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000090a0d94e. 00000100:00001000:1.0:1784949149.110208:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=21 v=5 (1 1 1 1) 00000020:00000001:0.0:1784949149.110219:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949149.110222:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00010000:00000040:0.0:1784949149.110225:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677568 00000100:00000001:1.0:1784949149.110227:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949149.110233:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000001:1.0:1784949149.110236:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949149.110241:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:0.0:1784949149.110246:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000017f9754b x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:55/0 lens 224/224 e 0 to 0 dl 1784949160 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949149.110248:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949149.110265:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949149.110272:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949149.110277:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949149.110285:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=16 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949149.110296:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949149.110297:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949149.110306:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1784949149.110307:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949149.110312:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949149.110314:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949149.110324:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949149.110326:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949149.110329:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949149.110331:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949149.110343:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949149.110351:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000100:00100000:1.0:1784949149.110358:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289677440:0@lo:400: 00000100:00000200:0.0:1784949149.110360:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289677568, offset 224 00000100:00000001:1.0:1784949149.110377:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949149.110380:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949149.110382:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949149.110389:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289677440/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949149.110399:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949149.110412:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:0.0:1784949149.110418:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3661:3661:249:4294967295] 192.168.202.137@tcp LPNI seq info [2358:2358:8:4294967295] 02000000:00000001:1.0:1784949149.110420:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949149.110427:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006841dfc4. 02000000:00000001:1.0:1784949149.110433:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949149.110440:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:0.0:1784949149.110442:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000040:1.0:1784949149.110444:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000800:00000200:0.0:1784949149.110455:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949149.110456:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949149.110463:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000635133da. 00000800:00000010:0.0:1784949149.110466:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000057c7ab4. 02000000:00000001:1.0:1784949149.110470:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949149.110475:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:0.0:1784949149.110479:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000010:1.0:1784949149.110481:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000800:00000200:0.0:1784949149.110494:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949149.110499:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000800:00000200:0.0:1784949149.110500:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000057c7ab4 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949149.110507:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949149.110511:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949149.110517:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949149.110519:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949149.110522:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949149.110524:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949149.110527:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949149.110533:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949149.110536:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949149.110538:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949149.110542:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949149.110549:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000017f9754b x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:55/0 lens 224/224 e 0 to 0 dl 1784949160 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949149.110561:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000017f9754b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289677568:12345-192.168.202.137@tcp:400: Request processed in 573us (1046us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949149.110570:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 763 00000100:00000040:0.0:1784949149.110573:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949149.110576:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949149.110578:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949149.110581:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c9127b8a. 00000020:00000010:0.0:1784949149.110587:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949149.110591:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000020:00000040:0.0:1784949149.110599:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949149.110602:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949149.110798:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949149.110807:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784949149.110808:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000057c7ab4. 00000800:00000001:0.0:1784949149.110821:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949149.110821:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949149.110830:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949149.110834:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949149.110842:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949149.110855:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949149.110857:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:2.0:1784949149.110864:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00000100:00000001:2.0:1784949149.110871:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:0.0:1784949149.110874:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643bf00 00000100:00000001:2.0:1784949149.110876:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784949149.110889:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x8fe1 [1] + 224 00000800:00000001:0.0:1784949149.110899:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949149.110941:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949149.110945:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949149.110951:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949149.110957:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949149.110961:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949149.110973:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949149.110994:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949149.111002:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949149.111005:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949149.111045:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949149.111050:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949149.112870:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.112871:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949149.112874:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000400:00000200:2.0:1784949149.112880:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784949149.112882:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949149.112885:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949149.112891:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949149.112903:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949149.112930:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949149.112932:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949149.112935:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949149.112937:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949149.112939:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949149.112941:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949149.112944:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949149.112951:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=16 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949149.112955:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=16 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949149.112957:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949149.112959:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949149.112961:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949149.112965:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949149.112974:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.112978:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949149.112980:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949149.112982:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949149.112983:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949149.112987:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949149.112995:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949149.112998:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949149.112999:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949149.113004:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949149.113013:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289677568:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949149.113018:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949149.113021:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949149.113023:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289677568/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949149.113031:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949149.113033:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949149.113035:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000008fae13bf. 02000000:00000001:2.0:1784949149.113040:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949149.113042:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949149.113044:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949149.113046:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949149.113049:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003a8a98bc. 02000000:00000001:2.0:1784949149.113051:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949149.113054:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949149.113055:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949149.113061:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949149.113062:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949149.113064:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949149.113069:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949149.113071:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949149.113074:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949149.113076:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949149.152024:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306088491, at 4306088492 00000100:00000040:3.0:1784949149.152035:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306088491, at 4306088492 00000100:00000010:2.0:1784949149.152039:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000addba683. 00000100:00000040:0.0:1784949149.152040:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306088491, at 4306088492 00000100:00000040:1.0:1784949149.152049:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306088491, at 4306088492 00000100:00000010:3.0:1784949149.152053:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003b6d02be. 00000100:00000010:0.0:1784949149.152058:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e36e3817. 00000100:00000010:1.0:1784949149.152071:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000fbfea82a. 00000800:00000100:2.0:1784949149.300049:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.28@tcp 00000800:00000010:2.0:1784949149.300055:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009ff40a64. 00000400:00000200:2.0:1784949149.300064:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949149.300074:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.28@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949149.300085:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11422 health :900 00000400:00000200:2.0:1784949149.300092:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949149.300097:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.28@tcp: 1 00000400:00000200:2.0:1784949149.300103:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.28@tcp: -114 00000400:00000200:1.0:1784949149.300171:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949149.300183:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.28@tcp(00000000e7299d92) state 0x840 00000400:00000001:1.0:1784949149.300190:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949149.300192:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949149.300194:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:1.0:1784949149.300197:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000400:00000200:1.0:1784949149.300203:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949149.300208:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.28@tcp: -114 00000400:00000010:1.0:1784949149.300213:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000038ac69a2. 00000400:00000200:1.0:1784949149.300220:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.28@tcp:-110 00000400:00000200:1.0:1784949149.300223:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.28@tcp(00000000e7299d92) state 0x2040 rc -110 00000400:00000200:1.0:1784949149.300227:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.28@tcp 00000400:00000200:3.0:1784949149.300309:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.28@tcp NID 111.222.142.28@tcp: -110. discovery complete 00000400:00000200:3.0:1784949149.300321:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.142.28@tcp primary NID 111.222.142.28@tcp rc -110 00000100:00000010:3.0:1784949149.300347:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000e8ecb15f. 00000100:00000001:3.0:1784949149.300383:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784949149.300388:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000e8ecb15f refcount 1 to 111.222.142.28@tcp 00010000:00000010:3.0:1784949149.300396:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000083930ef2. 00010000:00080000:3.0:1784949149.300456:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.142.28@tcp at tail 00010000:00000001:3.0:1784949149.300459:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949149.300464:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949149.300477:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 00000000980961c7. 00000100:00000400:3.0:1784949149.300482:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.173.29@tcp: rc = -75 00000100:00000040:3.0:1784949149.312416:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.173.29@tcp cnt=32 00000400:00000200:3.0:1784949149.312421:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784949149.312490:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784949149.312495:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784949149.312499:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784949149.312502:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784949149.312510:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784949149.312520:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.312523:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.312527:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784949149.312529:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784949149.312533:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.312552:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784949149.312556:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.312560:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.312564:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.312568:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784949149.312574:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784949149.312580:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.312589:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784949149.312591:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.312592:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.312596:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.312599:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784949149.312601:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784949149.312605:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.312612:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784949149.312614:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.312616:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.312619:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.312621:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784949149.312624:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:3.0:1784949149.312628:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.312636:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:3.0:1784949149.312639:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949149.312653:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949149.312653:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.312663:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.312667:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784949149.312669:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:2.0:1784949149.312679:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784949149.312679:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.312690:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784949149.312694:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.312703:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.312707:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000010:2.0:1784949149.312708:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000008fae13bf. 00000400:00000200:3.0:1784949149.312711:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:3.0:1784949149.312738:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000010:2.0:1784949149.312742:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000200:3.0:1784949149.312750:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.312760:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000200:3.0:1784949149.312762:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949149.312768:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b9d99f33. 00000400:00000200:3.0:1784949149.312768:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.312779:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.312785:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:2.0:1784949149.312791:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949149.312793:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:3.0:1784949149.312802:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.312814:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:2.0:1784949149.312821:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.312827:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.312833:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.312838:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:2.0:1784949149.312842:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3662:3662:250:4294967295] 111.222.173.1@tcp LPNI seq info [52:52:8:4294967295] 00000400:00000200:2.0:1784949149.312849:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:2.0:1784949149.312857:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.312864:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000800:00000010:2.0:1784949149.312871:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fb9675a2. 00000400:00000200:3.0:1784949149.312872:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784949149.312880:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.312883:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949149.312887:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949149.312896:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.312900:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000800:00000200:2.1:1784949149.312905:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949149.312962:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:2.0:1784949149.312970:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:3.0:1784949149.312972:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.312982:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:3.0:1784949149.312988:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:3.0:1784949149.312994:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.313005:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949149.313011:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784949149.313017:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313023:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784949149.313027:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000010:2.0:1784949149.313028:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006b8bfa56. 00000400:00000200:3.0:1784949149.313035:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949149.313038:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:3.0:1784949149.313045:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:3.0:1784949149.313049:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949149.313057:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000064d54234. 00000400:00000200:3.0:1784949149.313058:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.313068:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:2.0:1784949149.313078:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949149.313079:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784949149.313086:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:2.0:1784949149.313092:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784949149.313096:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.313104:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.313109:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.313112:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.313116:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:2.0:1784949149.313119:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3663:3663:249:4294967295] 111.222.142.1@tcp LPNI seq info [50:50:8:4294967295] 00000400:00000200:2.0:1784949149.313128:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:2.0:1784949149.313134:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.313142:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000800:00000010:2.0:1784949149.313147:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f95b4924. 00000400:00000200:3.0:1784949149.313148:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000200:3.0:1784949149.313154:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.313159:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949149.313165:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949149.313170:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313176:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000800:00000200:2.1:1784949149.313182:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949149.313185:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000200:2.0:1784949149.313197:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:3.0:1784949149.313199:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.313208:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:3.0:1784949149.313213:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:3.0:1784949149.313219:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.313224:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949149.313229:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784949149.313234:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313241:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784949149.313244:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000010:2.0:1784949149.313245:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005cc3cd5d. 00000400:00000200:3.0:1784949149.313252:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949149.313254:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000400:00000200:3.0:1784949149.313265:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784949149.313268:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949149.313273:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004f0f2833. 00000400:00000200:3.0:1784949149.313274:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.313287:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:2.0:1784949149.313294:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784949149.313298:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:2.0:1784949149.313305:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:3.0:1784949149.313313:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:2.0:1784949149.313315:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.313324:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949149.313332:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.313333:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.313343:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:2.0:1784949149.313348:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3664:3664:248:4294967295] 111.222.173.2@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:2.0:1784949149.313353:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:2.0:1784949149.313358:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.313363:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:2.0:1784949149.313367:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002f91e44d. 00000400:00000200:3.0:1784949149.313375:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000800:00000200:2.0:1784949149.313376:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949149.313387:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.1:1784949149.313393:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949149.313395:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.313408:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313411:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:3.0:1784949149.313413:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:3.0:1784949149.313419:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.313424:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:3.0:1784949149.313431:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:2.0:1784949149.313435:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:3.0:1784949149.313438:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.313447:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949149.313450:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:3.0:1784949149.313459:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313464:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:3.0:1784949149.313467:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000010:2.0:1784949149.313468:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000d531934e. 00000400:00000200:3.0:1784949149.313472:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949149.313478:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000400:00000200:3.0:1784949149.313484:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784949149.313488:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949149.313493:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006aad92e0. 00000400:00000200:3.0:1784949149.313494:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.313503:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:2.0:1784949149.313510:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949149.313513:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:3.0:1784949149.313520:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:2.0:1784949149.313525:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:3.0:1784949149.313527:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.313539:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.313544:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.313546:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.313550:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:2.0:1784949149.313554:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3665:3665:247:4294967295] 111.222.142.2@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:2.0:1784949149.313559:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:2.0:1784949149.313572:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.313580:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949149.313585:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000800:00000010:2.0:1784949149.313586:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000de6364cf. 00000400:00000200:3.0:1784949149.313591:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.313597:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949149.313603:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949149.313609:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000800:00000200:2.0:1784949149.313612:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 10 secs later) 00000400:00000200:3.0:1784949149.313616:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:3.0:1784949149.313624:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000800:00000200:2.1:1784949149.313632:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949149.313637:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949149.313642:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 10 secs later) 00000400:00000200:3.0:1784949149.313647:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:3.0:1784949149.313653:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.313657:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000100:2.0:1784949149.313660:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949149.313668:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000800:00000010:2.0:1784949149.313669:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000de6364cf. 00000400:00000200:3.0:1784949149.313673:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784949149.313679:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:2.0:1784949149.313684:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949149.313685:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.313695:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:3.0:1784949149.313699:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.313705:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949149.313706:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949149.313734:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313743:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:2.0:1784949149.313747:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949149.313747:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:3.0:1784949149.313755:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.313762:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:3.0:1784949149.313767:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:3.0:1784949149.313772:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949149.313783:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:3.0:1784949149.313792:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.313800:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313802:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784949149.313805:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:3.0:1784949149.313811:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.313817:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:3.0:1784949149.313825:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:2.0:1784949149.313826:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 rc 0 00000400:00000200:3.0:1784949149.313833:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949149.313845:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000010:2.0:1784949149.313855:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006200396a. 00000400:00000010:2.0:1784949149.313860:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000010:2.0:1784949149.313864:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a816c742. 00000400:00000200:2.0:1784949149.313868:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:2.0:1784949149.313875:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:2.0:1784949149.313879:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.313882:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.313885:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.313888:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:2.0:1784949149.313892:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3666:3666:247:4294967295] 111.222.173.3@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:2.0:1784949149.313899:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:2.0:1784949149.313943:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.313951:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:2.0:1784949149.313956:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000de6364cf. 00000800:00000200:2.0:1784949149.313963:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000800:00000200:2.1:1784949149.313970:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:2.0:1784949149.313975:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784949149.313976:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949149.313983:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:3.0:1784949149.313993:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.313997:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:2.0:1784949149.314005:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784949149.314005:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:3.0:1784949149.314017:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949149.314018:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000859865aa. 00000400:00000010:2.0:1784949149.314025:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000400:00000200:3.0:1784949149.314029:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:3.0:1784949149.314034:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949149.314040:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000074e9a075. 00000400:00000200:3.0:1784949149.314041:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.314052:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:2.0:1784949149.314061:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:3.0:1784949149.314063:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:3.0:1784949149.314070:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:2.0:1784949149.314076:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:3.0:1784949149.314083:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.314089:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.314095:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.314097:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.314100:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:2.0:1784949149.314103:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3667:3667:246:4294967295] 111.222.173.4@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:2.0:1784949149.314108:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:2.0:1784949149.314114:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.314119:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000800:00000010:2.0:1784949149.314123:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e1f7df0e. 00000400:00000200:3.0:1784949149.314129:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000800:00000200:2.0:1784949149.314133:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784949149.314139:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.1:1784949149.314146:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784949149.314149:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.314162:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.314167:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:3.0:1784949149.314170:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:2.0:1784949149.314177:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:3.0:1784949149.314182:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.314188:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784949149.314205:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784949149.314210:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.314219:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949149.314224:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000200:3.0:1784949149.314233:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.314241:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000010:2.0:1784949149.314247:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001799e5ef. 00000400:00000200:3.0:1784949149.314247:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000010:2.0:1784949149.314258:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000200:3.0:1784949149.314261:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949149.314272:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e4a8defd. 00000400:00000200:3.0:1784949149.314279:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:2.0:1784949149.314288:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:3.0:1784949149.314290:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.314301:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.314309:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.314314:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:2.0:1784949149.314320:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:3.0:1784949149.314322:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:2.0:1784949149.314331:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949149.314337:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.314339:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.314349:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.314354:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:2.0:1784949149.314357:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3668:3668:245:4294967295] 111.222.142.4@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:2.0:1784949149.314362:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:2.0:1784949149.314368:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.314373:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:2.0:1784949149.314378:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005523afda. 00000400:00000200:3.0:1784949149.314383:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:3.0:1784949149.314389:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784949149.314390:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:3.0:1784949149.314395:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949149.314397:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 10 secs later) 00000400:00000200:3.0:1784949149.314407:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000800:00000200:2.1:1784949149.314417:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:3.0:1784949149.314418:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000800:00000200:2.1:1784949149.314429:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 10 secs later) 00000800:00000100:2.0:1784949149.314440:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.4@tcp 00000800:00000010:2.0:1784949149.314443:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005523afda. 00000400:00000200:2.0:1784949149.314450:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949149.314454:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949149.314459:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949149.314462:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:2.0:1784949149.314466:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:2.0:1784949149.314473:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:2.0:1784949149.314476:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 rc 0 00000400:00000200:2.0:1784949149.314481:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:2.0:1784949149.314486:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:2.0:1784949149.314490:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000200:3.0:1784949149.314491:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000010:2.0:1784949149.314495:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000775d3d92. 00000400:00000200:3.0:1784949149.314503:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.314505:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:3.0:1784949149.314524:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784949149.314528:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949149.314537:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:2.0:1784949149.314542:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.314545:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.314548:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.314551:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:2.0:1784949149.314554:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3669:3669:245:4294967295] 111.222.142.5@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:2.0:1784949149.314559:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:2.0:1784949149.314566:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000400:00000200:3.0:1784949149.314571:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949149.314572:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:2.0:1784949149.314580:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005523afda. 00000400:00000200:3.0:1784949149.314582:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.314590:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000800:00000200:2.0:1784949149.314597:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:3.0:1784949149.314600:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:3.0:1784949149.314615:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949149.314616:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:2.0:1784949149.314630:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:3.0:1784949149.314631:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:2.0:1784949149.314640:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:3.0:1784949149.314641:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.314650:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.314656:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.314658:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000200:3.0:1784949149.314661:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:2.0:1784949149.314667:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000200:3.0:1784949149.314673:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.314683:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:3.0:1784949149.314685:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949149.314687:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f1904ca. 00000400:00000200:3.0:1784949149.314688:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.314697:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000010:2.0:1784949149.314701:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000200:3.0:1784949149.314702:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:3.0:1784949149.314708:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000010:2.0:1784949149.314741:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000473e84ce. 00000400:00000200:3.0:1784949149.314743:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949149.314760:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:2.0:1784949149.314764:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:3.0:1784949149.314765:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.314776:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.314780:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949149.314783:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784949149.314786:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000200:3.0:1784949149.314797:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949149.314799:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:2.0:1784949149.314806:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.314811:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.314813:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.314817:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:2.0:1784949149.314820:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3670:3670:244:4294967295] 111.222.173.6@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:2.0:1784949149.314827:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:2.0:1784949149.314833:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.314841:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000400:00000200:3.0:1784949149.314847:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000800:00000010:2.0:1784949149.314850:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fbc83221. 00000400:00000200:3.0:1784949149.314851:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949149.314857:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949149.314869:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000800:00000200:2.0:1784949149.314872:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:3.0:1784949149.314874:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000100:00000010:3.0:1784949149.314883:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 00000000980961c7. 00000800:00000200:2.1:1784949149.314891:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000020:00000001:3.0:1784949149.314900:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00010000:00000001:3.0:1784949149.314933:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000100:00000001:3.0:1784949149.314939:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000010:3.0:1784949149.314948:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000a5bcc72e. 00000400:00000200:2.0:1784949149.314949:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:2.0:1784949149.314959:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:3.0:1784949149.314963:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000a5bcc72e nid 111.222.173.29@tcp 00000400:00000200:2.0:1784949149.314971:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:3.0:1784949149.314972:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000002b896c5a. 00000400:00000200:3.0:1784949149.314982:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 000000002b896c5a net tcp 00000400:00000010:2.0:1784949149.314983:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dcf1bf81. 00000400:00000010:3.0:1784949149.314989:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000009713a832. 00000400:00000010:2.0:1784949149.314993:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000400:00000200:3.0:1784949149.315001:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 000000009713a832 nid 111.222.173.29@tcp 00000400:00000010:2.0:1784949149.315005:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000062d84ae9. 00000400:00000200:2.0:1784949149.315021:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:3.0:1784949149.315022:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.173.29@tcp NID 111.222.173.29@tcp flags 0x0 00000400:00000200:3.0:1784949149.315033:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.173.29@tcp: 0 00000400:00000200:2.0:1784949149.315039:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:2.0:1784949149.315047:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315052:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315056:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315059:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:2.0:1784949149.315064:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3671:3671:243:4294967295] 111.222.142.6@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:2.0:1784949149.315070:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:2.0:1784949149.315077:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315083:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000400:00000200:3.0:1784949149.315089:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.29@tcp: 0 00000800:00000010:2.0:1784949149.315090:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fd30a9ad. 00000400:00000200:3.0:1784949149.315095:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784949149.315104:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.0:1784949149.315108:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 10 secs later) 00000800:00000200:2.1:1784949149.315114:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784949149.315138:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 10 secs later) 00000800:00000100:2.0:1784949149.315146:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.6@tcp 00000800:00000010:2.0:1784949149.315149:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fd30a9ad. 00000400:00000200:2.0:1784949149.315154:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949149.315158:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949149.315162:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949149.315164:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:2.0:1784949149.315168:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:2.0:1784949149.315176:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:2.0:1784949149.315178:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 rc 0 00000400:00000200:2.0:1784949149.315182:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:2.0:1784949149.315187:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000be26f34f. 00000400:00000010:2.0:1784949149.315190:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000400:00000010:2.0:1784949149.315193:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000060798c14. 00000400:00000200:2.0:1784949149.315197:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:2.0:1784949149.315202:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:2.0:1784949149.315205:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315208:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315211:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315214:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:2.0:1784949149.315217:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3672:3672:243:4294967295] 111.222.173.7@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784949149.315222:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:2.0:1784949149.315229:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315233:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:2.0:1784949149.315236:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fd30a9ad. 00000800:00000200:2.0:1784949149.315241:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784949149.315245:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:2.0:1784949149.315248:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:2.0:1784949149.315250:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000400:00000200:2.0:1784949149.315253:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:2.0:1784949149.315257:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006841197d. 00000400:00000010:2.0:1784949149.315259:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000492bb2ef. 00000400:00000010:2.0:1784949149.315263:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a7d88830. 00000400:00000200:2.0:1784949149.315266:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:2.0:1784949149.315270:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:2.0:1784949149.315273:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315275:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315277:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315279:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:2.0:1784949149.315282:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3673:3673:242:4294967295] 111.222.173.8@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784949149.315286:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:2.0:1784949149.315294:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315301:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784949149.315306:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005c3de731. 00000800:00000200:2.0:1784949149.315312:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784949149.315316:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:2.0:1784949149.315335:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:2.0:1784949149.315338:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:2.0:1784949149.315342:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:2.0:1784949149.315346:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000033c25eed. 00000400:00000010:2.0:1784949149.315348:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000400:00000010:2.0:1784949149.315352:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000049855afc. 00000400:00000200:2.0:1784949149.315357:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:2.0:1784949149.315365:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:2.0:1784949149.315372:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315377:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315382:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315386:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:2.0:1784949149.315390:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3674:3674:241:4294967295] 111.222.142.8@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:2.0:1784949149.315395:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:2.0:1784949149.315407:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315414:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784949149.315419:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000313a63a1. 00000800:00000200:2.0:1784949149.315425:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.0:1784949149.315428:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 10 secs later) 00000800:00000200:2.1:1784949149.315434:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784949149.315436:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 10 secs later) 00000800:00000100:2.0:1784949149.315442:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784949149.315444:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000313a63a1. 00000400:00000200:2.0:1784949149.315448:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949149.315452:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949149.315456:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949149.315458:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:2.0:1784949149.315463:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:2.0:1784949149.315472:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:2.0:1784949149.315475:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 rc 0 00000400:00000200:2.0:1784949149.315480:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:2.0:1784949149.315488:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f0621080. 00000400:00000010:2.0:1784949149.315491:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000400:00000010:2.0:1784949149.315495:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c3fd57c9. 00000400:00000200:2.0:1784949149.315499:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:2.0:1784949149.315503:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:2.0:1784949149.315506:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315509:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315511:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315514:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:2.0:1784949149.315517:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3675:3675:241:4294967295] 111.222.142.9@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:2.0:1784949149.315522:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:2.0:1784949149.315528:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315533:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:2.0:1784949149.315535:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000313a63a1. 00000800:00000200:2.0:1784949149.315540:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784949149.315543:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:2.0:1784949149.315547:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:2.0:1784949149.315550:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:2.0:1784949149.315556:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:2.0:1784949149.315563:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001a4eba99. 00000400:00000010:2.0:1784949149.315567:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000400:00000010:2.0:1784949149.315571:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d54d9bc5. 00000400:00000200:2.0:1784949149.315575:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:2.0:1784949149.315580:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:2.0:1784949149.315583:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315586:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315588:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315591:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:2.0:1784949149.315594:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3676:3676:240:4294967295] 111.222.173.10@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:2.0:1784949149.315599:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:2.0:1784949149.315605:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315609:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784949149.315613:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000800:00000200:2.0:1784949149.315617:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784949149.315621:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:2.0:1784949149.315638:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:2.0:1784949149.315642:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:2.0:1784949149.315647:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000001:2.0:1784949149.315650:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.315651:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949149.315654:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000010:2.0:1784949149.315656:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000400:00000200:2.0:1784949149.315661:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949149.315667:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:2.0:1784949149.315671:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000d531934e. 00000400:00000200:2.0:1784949149.315680:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-5 00000400:00000200:2.0:1784949149.315687:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -5 00000400:00000200:2.0:1784949149.315695:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:2.0:1784949149.315702:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:2.0:1784949149.315710:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000d531934e. 00000400:00000010:2.0:1784949149.315738:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000400:00000010:2.0:1784949149.315747:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b23f11e5. 00000400:00000200:2.0:1784949149.315755:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:2.0:1784949149.315765:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:2.0:1784949149.315771:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315776:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315782:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315788:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:2.0:1784949149.315794:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3677:3677:239:4294967295] 111.222.142.10@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:2.0:1784949149.315803:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:2.0:1784949149.315809:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315815:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:2.0:1784949149.315820:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000800:00000200:2.0:1784949149.315826:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.0:1784949149.315828:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 10 secs later) 00000800:00000200:2.1:1784949149.315834:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784949149.315836:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 10 secs later) 00000800:00000100:2.0:1784949149.315841:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.10@tcp 00000800:00000010:2.0:1784949149.315843:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:2.0:1784949149.315847:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949149.315851:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949149.315855:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949149.315858:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:2.0:1784949149.315862:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:2.0:1784949149.315871:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:2.0:1784949149.315875:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 rc 0 00000400:00000200:2.0:1784949149.315882:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:2.0:1784949149.315888:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000da755428. 00000400:00000010:2.0:1784949149.315892:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000010:2.0:1784949149.315896:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000084f96ec7. 00000400:00000200:2.0:1784949149.315900:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:2.0:1784949149.315935:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:2.0:1784949149.315941:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.315945:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.315951:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.315955:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:2.0:1784949149.315962:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3678:3678:239:4294967295] 111.222.173.11@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:2.0:1784949149.315969:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:2.0:1784949149.315982:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.315990:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784949149.315995:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000800:00000200:2.0:1784949149.316002:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784949149.316008:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:2.0:1784949149.316011:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:2.0:1784949149.316014:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:2.0:1784949149.316018:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:2.0:1784949149.316025:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eb983bef. 00000400:00000010:2.0:1784949149.316028:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000400:00000010:2.0:1784949149.316032:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009cb36717. 00000400:00000200:2.0:1784949149.316035:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:2.0:1784949149.316040:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:2.0:1784949149.316043:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316046:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316049:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316052:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:2.0:1784949149.316055:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3679:3679:238:4294967295] 111.222.142.11@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:2.0:1784949149.316060:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:2.0:1784949149.316065:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316071:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:2.0:1784949149.316074:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c2b2d4f6. 00000800:00000200:2.0:1784949149.316079:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784949149.316083:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:2.0:1784949149.316086:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:2.0:1784949149.316088:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:2.0:1784949149.316091:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:2.0:1784949149.316095:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003cdea007. 00000400:00000010:2.0:1784949149.316097:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000400:00000010:2.0:1784949149.316101:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ccf4a53f. 00000400:00000200:2.0:1784949149.316104:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:2.0:1784949149.316107:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:2.0:1784949149.316110:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316112:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316114:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316117:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:2.0:1784949149.316120:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3680:3680:237:4294967295] 111.222.173.12@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:2.0:1784949149.316125:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:2.0:1784949149.316131:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316136:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784949149.316139:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000800:00000200:2.0:1784949149.316142:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784949149.316146:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:2.0:1784949149.316164:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:2.0:1784949149.316167:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:2.0:1784949149.316170:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:2.0:1784949149.316174:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001df3b40e. 00000400:00000010:2.0:1784949149.316177:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000400:00000010:2.0:1784949149.316182:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000521edb81. 00000400:00000200:2.0:1784949149.316186:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:2.0:1784949149.316193:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:2.0:1784949149.316199:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316203:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316206:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316211:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:2.0:1784949149.316216:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3681:3681:236:4294967295] 111.222.142.12@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:2.0:1784949149.316222:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:2.0:1784949149.316228:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316233:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784949149.316239:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000800:00000200:2.0:1784949149.316243:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.0:1784949149.316246:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 10 secs later) 00000800:00000200:2.1:1784949149.316252:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784949149.316255:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 10 secs later) 00000800:00000100:2.0:1784949149.316261:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784949149.316263:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:2.0:1784949149.316267:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949149.316270:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949149.316275:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949149.316278:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:2.0:1784949149.316283:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:2.0:1784949149.316290:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:2.0:1784949149.316293:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 rc 0 00000400:00000200:2.0:1784949149.316296:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:2.0:1784949149.316299:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.316301:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949149.316304:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:2.0:1784949149.316307:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000400:00000200:2.0:1784949149.316311:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949149.316313:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:2.0:1784949149.316316:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001799e5ef. 00000400:00000200:2.0:1784949149.316325:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-5 00000400:00000200:2.0:1784949149.316328:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -5 00000400:00000200:2.0:1784949149.316332:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:2.0:1784949149.316335:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:2.0:1784949149.316340:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001799e5ef. 00000400:00000010:2.0:1784949149.316343:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000010:2.0:1784949149.316347:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000cd70637b. 00000400:00000200:2.0:1784949149.316350:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:2.0:1784949149.316354:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:2.0:1784949149.316357:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316360:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316362:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316365:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:2.0:1784949149.316369:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3682:3682:236:4294967295] 111.222.142.13@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1784949149.316373:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:2.0:1784949149.316379:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316383:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:2.0:1784949149.316386:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000800:00000200:2.0:1784949149.316390:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784949149.316394:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:2.0:1784949149.316397:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:2.0:1784949149.316399:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:2.0:1784949149.316402:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:2.0:1784949149.316406:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f11303f8. 00000400:00000010:2.0:1784949149.316410:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000400:00000010:2.0:1784949149.316416:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000455b4195. 00000400:00000200:2.0:1784949149.316421:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:2.0:1784949149.316430:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:2.0:1784949149.316434:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316437:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316439:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316442:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:2.0:1784949149.316445:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3683:3683:235:4294967295] 111.222.173.14@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:2.0:1784949149.316450:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:2.0:1784949149.316456:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316460:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784949149.316463:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000800:00000200:2.0:1784949149.316468:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784949149.316474:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:2.0:1784949149.316491:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:2.0:1784949149.316494:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:2.0:1784949149.316498:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:2.0:1784949149.316502:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002fbb6924. 00000400:00000010:2.0:1784949149.316505:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000400:00000010:2.0:1784949149.316510:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b2848ca3. 00000400:00000200:2.0:1784949149.316515:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:2.0:1784949149.316523:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:2.0:1784949149.316527:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316531:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316534:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316537:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:2.0:1784949149.316540:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3684:3684:234:4294967295] 111.222.142.14@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:2.0:1784949149.316546:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:2.0:1784949149.316555:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316560:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:2.0:1784949149.316565:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000800:00000200:2.0:1784949149.316569:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.0:1784949149.316571:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 10 secs later) 00000800:00000200:2.1:1784949149.316576:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.1:1784949149.316578:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 10 secs later) 00000800:00000100:2.0:1784949149.316583:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.14@tcp 00000800:00000010:2.0:1784949149.316585:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:2.0:1784949149.316588:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949149.316591:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949149.316594:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949149.316596:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:2.0:1784949149.316600:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:2.0:1784949149.316605:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:2.0:1784949149.316607:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 rc 0 00000400:00000200:2.0:1784949149.316611:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:2.0:1784949149.316614:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000047e19e91. 00000400:00000010:2.0:1784949149.316617:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000010:2.0:1784949149.316622:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002085654e. 00000400:00000200:2.0:1784949149.316626:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:2.0:1784949149.316631:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:2.0:1784949149.316636:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316639:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316641:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316645:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:2.0:1784949149.316649:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3685:3685:234:4294967295] 111.222.173.15@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:2.0:1784949149.316655:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:2.0:1784949149.316665:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316672:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784949149.316678:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000800:00000200:2.0:1784949149.316685:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784949149.316692:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:2.0:1784949149.316699:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:2.0:1784949149.316704:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:2.0:1784949149.316711:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:2.0:1784949149.316743:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef8e57c8. 00000400:00000010:2.0:1784949149.316750:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000010:2.0:1784949149.316758:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000079caf10d. 00000400:00000200:2.0:1784949149.316766:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:2.0:1784949149.316776:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:2.0:1784949149.316782:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949149.316788:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316794:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949149.316799:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:2.0:1784949149.316805:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3686:3686:233:4294967295] 111.222.142.15@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:2.0:1784949149.316811:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:2.0:1784949149.316820:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316828:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:2.0:1784949149.316834:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000800:00000200:2.0:1784949149.316839:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784949149.316846:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:2.0:1784949149.316850:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:2.0:1784949149.316852:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:2.0:1784949149.316856:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.29@tcp(00000000a5bcc72e) state 0x40 00000400:00000010:2.0:1784949149.316864:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000075318cf7. 00000400:00000010:2.0:1784949149.316869:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000400:00000010:2.0:1784949149.316875:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000029ca9b8d. 00000400:00000200:2.0:1784949149.316879:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.29@tcp 00000400:00000200:2.0:1784949149.316886:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.173.29@tcp local destination 00000400:00000200:2.0:1784949149.316891:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949149.316895:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.173.29@tcp 00000400:00000200:2.0:1784949149.316900:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.173.29@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:2.0:1784949149.316905:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3687:3687:232:4294967295] 111.222.173.29@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1784949149.316955:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.29@tcp 00000400:00000200:2.0:1784949149.316963:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.29@tcp(111.222.173.29@tcp:111.222.173.29@tcp) : GET try# 0 00000800:00000200:2.0:1784949149.316974:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.29@tcp 00000800:00000010:2.0:1784949149.316980:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000800:00000010:2.0:1784949149.316988:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000efb93606. 00000800:00000010:2.0:1784949149.316993:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 000000001d10ae9a. 00000800:00000200:2.0:1784949149.316999:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000efb93606] -> 12345-111.222.173.29@tcp (2) 00000800:00000200:2.1:1784949149.317008:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000efb93606] -> 12345-111.222.173.29@tcp (2) 00000400:00000200:2.0:1784949149.317024:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.29@tcp 00000400:00000200:2.0:1784949149.317028:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.29@tcp(00000000a5bcc72e) state 0x240 rc 0 00000400:00000200:2.0:1784949149.317033:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 00000400:00000001:2.0:1784949149.317037:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.317040:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949149.317042:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784949149.317045:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000400:00000200:2.0:1784949149.317052:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949149.317062:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:2.0:1784949149.317066:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dcf1bf81. 00000400:00000200:2.0:1784949149.317074:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-5 00000400:00000200:2.0:1784949149.317077:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -5 00000400:00000200:2.0:1784949149.317083:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:2.0:1784949149.317092:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:2.0:1784949149.317095:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.317097:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949149.317099:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000400:00000010:2.0:1784949149.317101:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000400:00000200:2.0:1784949149.317106:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949149.317114:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:2.0:1784949149.317117:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000033c25eed. 00000400:00000200:2.0:1784949149.317121:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-5 00000400:00000200:2.0:1784949149.317124:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -5 00000400:00000200:2.0:1784949149.317127:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000400:00000200:2.0:1784949149.317132:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:2.0:1784949149.317134:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.317135:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949149.317137:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000010:2.0:1784949149.317139:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000400:00000200:2.0:1784949149.317143:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949149.317147:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:2.0:1784949149.317150:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000d531934e. 00000400:00000200:2.0:1784949149.317157:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-5 00000400:00000200:2.0:1784949149.317161:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -5 00000400:00000200:2.0:1784949149.317166:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000400:00000200:2.0:1784949149.317173:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:2.0:1784949149.317177:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.317178:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949149.317179:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:2.0:1784949149.317181:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000400:00000200:2.0:1784949149.317184:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949149.317186:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:2.0:1784949149.317188:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001df3b40e. 00000400:00000200:2.0:1784949149.317192:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-5 00000400:00000200:2.0:1784949149.317197:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -5 00000400:00000200:2.0:1784949149.317200:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000400:00000200:2.0:1784949149.317204:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:2.0:1784949149.317207:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949149.317209:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949149.317211:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:2.0:1784949149.317213:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000400:00000200:2.0:1784949149.317216:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949149.317220:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:2.0:1784949149.317222:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002fbb6924. 00000400:00000200:2.0:1784949149.317229:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-5 00000400:00000200:2.0:1784949149.317233:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -5 00000400:00000200:2.0:1784949149.317236:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00010000:00000001:2.0:1784949149.940031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949149.940039:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949149.940047:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949149.940049:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949149.940177:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949149.940184:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949149.940189:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949150.132026:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949150.132041:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949150.132044:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949150.132050:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949150.132054:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949150.132059:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949150.132062:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949150.324096:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949150.324107:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000951ab34f. 00000400:00000010:3.0:1784949150.324119:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:3.0:1784949150.324126:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000400:00000010:3.0:1784949150.324139:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000189c3332. 00000400:00000200:3.0:1784949150.324145:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949150.324165:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949150.324175:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949150.324181:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3688:3688:231:4294967295] 192.168.202.137@tcp LPNI seq info [2359:2359:8:4294967295] 00000400:00000200:3.0:1784949150.324191:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000800:00000100:1.0:1784949150.324210:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.5@tcp 00000400:00000200:3.0:1784949150.324212:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000010:1.0:1784949150.324225:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b84187a1. 00000800:00000200:3.0:1784949150.324232:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949150.324239:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000010:3.0:1784949150.324240:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000400:00000200:1.0:1784949150.324256:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: NETWORK_TIMEOUT 00000800:00000200:3.0:1784949150.324263:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949150.324274:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11423 health :800 00000800:00000200:3.0:1784949150.324283:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:1.0:1784949150.324288:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000200:3.0:1784949150.324290:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949150.324299:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:1.0:1784949150.324306:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:0.0:1784949150.324396:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949150.324408:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2861 00000400:00000001:0.0:1784949150.324417:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949150.324421:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949150.324424:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000236111b7 00000400:00000010:0.0:1784949150.324427:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000236111b7. 00000400:00000200:0.0:1784949150.324433:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949150.324438:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:0.0:1784949150.324443:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001c9428b0. 00000400:00000200:0.0:1784949150.324453:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-110 00000400:00000200:0.0:1784949150.324457:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -110 00000400:00000200:0.0:1784949150.324462:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000800:00000200:2.0:1784949150.324487:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949150.324495:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:2.0:1784949150.324504:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949150.324512:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949150.324518:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949150.324524:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949150.324541:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949150.324555:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949150.324560:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949150.324566:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949150.324578:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949150.324594:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949150.324606:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949150.324618:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949150.324624:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3689:3689:232:4294967295] 192.168.202.137@tcp LPNI seq info [2360:2360:8:4294967295] 00000400:00000200:2.0:1784949150.324636:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949150.324642:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949150.324647:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ca1124f5. 00000800:00000200:2.0:1784949150.324653:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949150.324659:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949150.324663:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ca1124f5 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949150.324774:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949150.324820:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949150.324826:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ca1124f5. 00000400:00000200:2.0:1784949150.324831:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949150.324839:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949150.324840:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949150.324857:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949150.324863:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949150.324866:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949150.324871:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949150.324888:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949150.324932:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x905d 00000800:00000001:3.0:1784949150.324939:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949150.324950:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949150.324953:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949150.324957:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949150.324963:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:3.0:1784949150.324966:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000400:00000200:3.0:1784949150.324969:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949150.324973:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000951ab34f. 00000400:00000010:3.0:1784949150.324977:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:3.0:1784949150.324985:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949150.324988:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949150.964032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949150.964040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949150.964048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949150.964052:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949150.964056:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949150.964059:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949150.964063:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949150.964066:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949150.964068:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949150.964071:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949150.964198:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949150.964205:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949150.964209:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949151.156106:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949151.156120:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949151.156123:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949151.156129:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949151.156133:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949151.156138:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949151.156142:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949151.348057:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.3@tcp 00000800:00000010:1.0:1784949151.348065:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008746b1e6. 00000400:00000200:1.0:1784949151.348076:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949151.348084:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949151.348093:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11424 health :900 00000400:00000200:1.0:1784949151.348099:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949151.348104:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:1.0:1784949151.348109:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000800:00000100:1.0:1784949151.348139:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.9@tcp 00000800:00000010:1.0:1784949151.348142:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002aeee8e9. 00000400:00000200:1.0:1784949151.348147:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949151.348152:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949151.348158:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949151.348163:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:1.0:1784949151.348168:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:0.0:1784949151.348177:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949151.348192:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2861 00000400:00000001:0.0:1784949151.348202:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949151.348205:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949151.348209:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949151.348213:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000400:00000200:0.0:1784949151.348221:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949151.348226:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:0.0:1784949151.348232:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c73b3f00. 00000400:00000200:0.0:1784949151.348239:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-110 00000400:00000200:0.0:1784949151.348242:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -110 00000400:00000200:0.0:1784949151.348247:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00000400:00000200:0.0:1784949151.348251:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2861 00000400:00000001:0.0:1784949151.348254:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949151.348256:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949151.348257:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000400:00000010:0.0:1784949151.348259:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000400:00000200:0.0:1784949151.348262:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949151.348264:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:0.0:1784949151.348267:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000013f68999. 00000400:00000200:0.0:1784949151.348271:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:0.0:1784949151.348273:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000400:00000200:0.0:1784949151.348276:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00010000:00000001:2.0:1784949151.988034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949151.988042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949151.988050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949151.988052:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949151.988148:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949151.988156:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949151.988161:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949152.180032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949152.180048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949152.180051:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949152.180057:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949152.180061:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949152.180067:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949152.180071:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949152.372148:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:1.0:1784949152.372150:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.7@tcp 00000800:00000010:1.0:1784949152.372162:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000006487a147. 00000400:00000010:3.0:1784949152.372168:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000951ab34f. 00000400:00000200:1.0:1784949152.372176:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949152.372185:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a17860fa. 00000400:00000200:1.0:1784949152.372194:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784949152.372202:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000400:00000200:1.0:1784949152.372214:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11425 health :700 00000400:00000200:1.0:1784949152.372224:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:3.0:1784949152.372225:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000511e5266. 00000400:00000200:1.0:1784949152.372240:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:3.0:1784949152.372247:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949152.372257:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:0.0:1784949152.372391:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949152.372407:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949152.372418:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949152.372423:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3690:3690:234:4294967295] 192.168.202.137@tcp LPNI seq info [2361:2361:8:4294967295] 00000400:00000200:3.0:1784949152.372434:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949152.372444:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949152.372451:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949152.372461:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000400:00000200:0.0:1784949152.372470:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2861 00000800:00000100:1.0:1784949152.372482:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000400:00000001:0.0:1784949152.372485:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:3.0:1784949152.372495:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000001:0.0:1784949152.372496:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000010:1.0:1784949152.372499:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000034aacccc. 00000400:00000200:0.0:1784949152.372512:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000400:00000010:0.0:1784949152.372521:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000400:00000200:1.0:1784949152.372521:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000200:3.0:1784949152.372536:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:0.0:1784949152.372543:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949152.372547:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000800:00000200:3.0:1784949152.372549:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1784949152.372569:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:0.0:1784949152.372580:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002c1a3d41. 00000400:00000200:0.0:1784949152.372593:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-110 00000400:00000200:1.0:1784949152.372594:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949152.372604:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -110 00000400:00000200:1.0:1784949152.372608:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:1.0:1784949152.372621:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:0.0:1784949152.372636:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:0.0:1784949152.372655:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2861 00000400:00000001:0.0:1784949152.372661:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949152.372664:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949152.372667:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:0.0:1784949152.372671:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000400:00000200:0.0:1784949152.372677:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949152.372684:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:0.0:1784949152.372729:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e120de1c. 00000400:00000200:0.0:1784949152.372738:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:0.0:1784949152.372741:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:0.0:1784949152.372745:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000800:00000200:2.0:1784949152.372844:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949152.372852:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:2.0:1784949152.372862:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949152.372872:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949152.372873:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949152.372885:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949152.372890:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784949152.372891:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949152.372899:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949152.372934:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949152.372946:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949152.372966:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949152.372981:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949152.372996:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949152.373004:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3691:3691:236:4294967295] 192.168.202.137@tcp LPNI seq info [2362:2362:8:4294967295] 00000400:00000200:0.0:1784949152.373017:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949152.373024:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949152.373029:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000036e798e7. 00000800:00000200:0.0:1784949152.373037:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949152.373047:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949152.373051:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000036e798e7 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949152.373186:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949152.373224:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949152.373230:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949152.373234:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000036e798e7. 00000800:00000001:2.0:1784949152.373246:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949152.373247:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949152.373255:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949152.373257:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949152.373275:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949152.373275:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949152.373295:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949152.373312:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9061 00000800:00000001:2.0:1784949152.373320:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949152.373333:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949152.373336:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949152.373341:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949152.373348:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:2.0:1784949152.373351:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000400:00000200:2.0:1784949152.373354:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949152.373358:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000951ab34f. 00000400:00000010:2.0:1784949152.373361:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a17860fa. 00000800:00000001:2.0:1784949152.373369:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949152.373372:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949153.011984:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949153.011991:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949153.011996:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949153.011997:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949153.012051:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949153.012056:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949153.012060:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949153.203984:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949153.203996:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949153.203998:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949153.204003:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949153.204006:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949153.204009:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949153.204011:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949154.036043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949154.036052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949154.036060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949154.036061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949154.036129:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949154.036136:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949154.036141:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949154.228078:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949154.228089:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949154.228099:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000617daa1a. 00000020:00000040:0.0:1784949154.228107:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949154.228114:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949154.228118:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949154.228126:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949154.228136:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:2.0:1784949154.228139:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949154.228145:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.228153:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949154.228158:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949154.228161:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949154.228166:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949154.228173:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949154.228187:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949154.228190:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.228196:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949154.228198:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000617daa1a] to pc [ptlrpcd_00_01+1] req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949154.228206:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949154.228243:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949154.228248:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949154.228254:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949154.228259:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949154.228260:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949154.228268:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00100000:2.0:1784949154.228270:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000010:0.0:1784949154.228275:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949154.228276:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949154.228286:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949154.228288:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949154.228292:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949154.228294:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949154.228294:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949154.228298:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949154.228306:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.228308:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.228310:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949154.228319:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000820f02b3. 00000100:00000001:2.0:1784949154.228321:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949154.228328:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949154.228330:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949154.228347:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784949154.228348:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949154.228388:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949154.228392:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.228396:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784949154.228402:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_02+2] req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949154.228403:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949154.228414:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949154.228421:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949154.228428:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949154.228432:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289677696:0@lo:400: 00000100:00000001:2.0:1784949154.228439:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949154.228444:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949154.228447:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949154.228454:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949154.228455:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11431) 02000000:00000001:2.0:1784949154.228458:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949154.228469:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1784949154.228470:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000002fbb6924. 02000000:00000001:2.0:1784949154.228476:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.228477:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949154.228483:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:2.0:1784949154.228486:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000400:00000010:2.0:1784949154.228500:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000040:3.0:1784949154.228500:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949154.228517:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677696, portal 25 00000100:00000001:2.0:1784949154.228531:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949154.228532:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949154.228535:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554966272 : -115544154585344 : ffff96e9c7b5c700) 00000100:00000001:3.0:1784949154.228536:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949154.228550:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1784949154.228553:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949154.228559:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949154.228563:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949154.228571:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949154.228574:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949154.228589:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289677824:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949154.228596:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949154.228599:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949154.228606:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1784949154.228607:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949154.228619:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 02000000:00000001:3.0:1784949154.228619:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949154.228636:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000200:2.0:1784949154.228637:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289677696, offset 0 02000000:00000010:3.0:1784949154.228652:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000378c7327. 00000400:00000200:2.0:1784949154.228661:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784949154.228662:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949154.228695:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000400:00000010:3.0:1784949154.228708:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6ceb0f1. 00000400:00000200:2.0:1784949154.228715:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:3.0:1784949154.228715:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677824, portal 10 00000100:00000001:3.0:1784949154.228729:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949154.228733:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000400:00000200:2.0:1784949154.228743:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643bf80 00000100:00000040:3.0:1784949154.228749:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949154.228767:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8665 [2] + 0 00000100:00000001:3.0:1784949154.228775:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:2.0:1784949154.228784:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000010bdacaf. 00000100:00000040:3.0:1784949154.228790:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949154.228795:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000096db69f7. 00000400:00000200:2.0:1784949154.228796:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000059abd2b8 00000100:00000200:3.0:1784949154.228804:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289677824, offset 0 00000400:00000200:2.0:1784949154.228810:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949154.228814:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949154.228823:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949154.228835:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949154.228842:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949154.228847:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949154.228851:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949154.228854:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949154.228857:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3692:3692:236:4294967295] 192.168.202.137@tcp LPNI seq info [2363:2363:8:4294967295] 00000400:00000200:3.0:1784949154.228870:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949154.228877:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949154.228879:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000059abd2b8 00000400:00000010:2.0:1784949154.228884:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000059abd2b8. 00000800:00000010:3.0:1784949154.228885:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000100:00000001:2.0:1784949154.228959:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949154.228965:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949154.228973:0:337119:0:(events.c:337:request_in_callback()) incoming req@000000003bfcf238 x1871651289677696 msgsize 224 00000100:00100000:2.0:1784949154.228980:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949154.228984:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:3.0:1784949154.228992:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949154.229004:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949154.229016:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000800:00000200:3.0:1784949154.229017:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000010:2.0:1784949154.229021:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:2.0:1784949154.229032:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949154.229051:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949154.229053:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949154.229082:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949154.229088:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949154.229097:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.229098:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.229103:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1784949154.229108:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2193 usecs (3 jiffies) 00000100:00000001:2.0:1784949154.229112:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.229115:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306093572 00000100:00000001:2.0:1784949154.229121:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306093611 00000100:00000010:3.0:1784949154.229124:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000001:1.0:1784949154.229132:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949154.229136:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00100000:1.0:1784949154.229141:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677696 02000000:00000001:1.0:1784949154.229147:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949154.229151:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949154.229152:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306093572 00000100:00000001:1.0:1784949154.229157:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949154.229159:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:2.0:1784949154.229160:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306093572 02000000:00000001:1.0:1784949154.229175:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.229176:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949154.229179:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306093611 00000100:00100000:1.0:1784949154.229195:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677696 00000100:00000001:3.0:1784949154.229200:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306093572 00000020:00000001:1.0:1784949154.229202:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949154.229202:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306093572 00000020:00000040:1.0:1784949154.229213:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:3.0:1784949154.229214:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000029366a5. 00000100:00000001:2.0:1784949154.229216:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306093611 00000020:00000001:1.0:1784949154.229227:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949154.229237:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949154.229239:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000001:1.0:1784949154.229250:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949154.229255:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306093572 00000020:00000001:1.0:1784949154.229258:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949154.229259:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306093572 00000100:00000010:3.0:1784949154.229262:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000068c34307. 00000100:00000001:1.0:1784949154.229269:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949154.229277:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306093611 00000100:00000001:3.0:1784949154.229281:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000010:1.0:1784949154.229283:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec77eb13. 00000100:00000001:1.0:1784949154.229290:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949154.229292:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949154.229295:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306093572 00000800:00000200:0.0:1784949154.229300:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:1.0:1784949154.229315:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec77eb13. 00000800:00000010:0.0:1784949154.229329:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000100:00000001:2.0:1784949154.229330:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949154.229332:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949154.229339:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00000400:00000200:0.0:1784949154.229349:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949154.229356:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.229359:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949154.229362:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000329e608d. 00000100:00000001:2.0:1784949154.229364:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.229374:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949154.229376:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:1.0:1784949154.229379:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:2.0:1784949154.229380:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949154.229391:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1784949154.229397:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096db69f7 00000100:00000001:3.0:1784949154.229399:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:1.0:1784949154.229407:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000010:0.0:1784949154.229414:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000096db69f7. 00000100:00000001:3.0:1784949154.229417:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949154.229418:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949154.229429:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949154.229434:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949154.229436:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000001:2.0:1784949154.229443:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784949154.229451:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949154.229454:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949154.229456:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949154.229467:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949154.229482:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949154.229483:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949154.229489:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949154.229497:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949154.229515:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949154.229519:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949154.229524:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949154.229527:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949154.229546:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949154.229554:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784949154.229554:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c000 00000100:00000001:1.0:1784949154.229558:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949154.229566:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949154.229578:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 172176 00000100:00100000:1.0:1784949154.229587:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 788 00000100:00000001:3.0:1784949154.229590:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:2.0:1784949154.229593:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949154.229599:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949154.229601:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:2.0:1784949154.229607:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949154.229610:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949154.229612:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742039632 : -115538967511984 : ffff96eafce23c50) 00000100:00000001:3.0:1784949154.229632:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949154.229633:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784949154.229642:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003bfcf238 x1871651289677696/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:60/0 lens 224/0 e 0 to 0 dl 1784949165 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949154.229663:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949154.229667:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949154.229671:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949154.229701:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000010:1.0:1784949154.229704:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ec77eb13. 00000100:00000010:2.0:1784949154.229709:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000016685826. 00000100:00000001:1.0:1784949154.229711:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949154.229718:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000016685826 x1871651289677824 msgsize 224 00000020:00000010:1.0:1784949154.229719:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ec77eb13. 00000100:00100000:1.0:1784949154.229739:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003bfcf238 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289677696:12345-0@lo:400: 00000100:00100000:2.0:1784949154.229746:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:1.0:1784949154.229753:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677696 00000020:00000001:1.0:1784949154.229758:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949154.229763:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949154.229766:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949154.229769:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949154.229774:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784949154.229774:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:1.0:1784949154.229783:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000001:2.0:1784949154.229787:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949154.229788:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949154.229795:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:2.0:1784949154.229798:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949154.229800:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949154.229807:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949154.229812:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949154.229816:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949154.229820:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949154.229827:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000579609a4. 00000100:00000001:0.0:1784949154.229830:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:1.0:1784949154.229831:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949154.229837:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677824 00000100:00000001:1.0:1784949154.229839:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949154.229845:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:1.0:1784949154.229847:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949154.229850:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00010000:00000040:1.0:1784949154.229854:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677696 00000100:00000001:0.0:1784949154.229856:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949154.229866:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:0.0:1784949154.229868:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949154.229875:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677824 00010000:00000200:1.0:1784949154.229876:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003bfcf238 x1871651289677696/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:60/0 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949154.229882:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949154.229887:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949154.229894:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949154.229900:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949154.229942:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949154.229945:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949154.229950:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:1.0:1784949154.229954:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:0.0:1784949154.229956:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00010000:00000001:1.0:1784949154.229961:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949154.229963:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949154.229971:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00001000:1.0:1784949154.229979:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=26 v=5 (1 1 1 1) 00000020:00000010:0.0:1784949154.229982:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:1.0:1784949154.229990:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:0.0:1784949154.229992:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c343a817. 00000100:00000040:1.0:1784949154.230000:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000010:0.0:1784949154.230002:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949154.230007:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784949154.230013:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949154.230017:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949154.230018:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949154.230028:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949154.230029:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949154.230031:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:1.0:1784949154.230037:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949154.230049:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949154.230053:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000001:0.0:1784949154.230058:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:1.0:1784949154.230062:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289677696, offset 224 00000100:00000001:0.0:1784949154.230064:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949154.230071:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949154.230078:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949154.230091:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949154.230097:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949154.230107:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643bf80 00000400:00000200:1.0:1784949154.230120:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9065 [1] + 224 00000100:00000001:0.0:1784949154.230122:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949154.230129:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784949154.230135:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:0.0:1784949154.230145:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 764 00000100:00000040:0.0:1784949154.230152:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000200:1.0:1784949154.230154:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949154.230159:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562022144 : -115544147529472 : ffff96e9c8217100) 00000100:00000001:1.0:1784949154.230169:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784949154.230171:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000016685826 x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:60/0 lens 224/0 e 0 to 0 dl 1784949165 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949154.230184:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949154.230197:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949154.230204:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:0.0:1784949154.230210:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949154.230211:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949154.230217:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00100000:0.0:1784949154.230235:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000016685826 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289677824:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949154.230251:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677824 00000020:00000001:0.0:1784949154.230255:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949154.230259:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949154.230261:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949154.230264:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949154.230265:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1784949154.230267:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949154.230274:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949154.230278:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949154.230279:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000020:00000001:0.0:1784949154.230283:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000010:1.0:1784949154.230285:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000020:00000001:0.0:1784949154.230288:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949154.230294:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949154.230295:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949154.230302:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:0.0:1784949154.230303:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949154.230320:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949154.230321:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949154.230325:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949154.230331:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000aef87956. 00000100:00000040:1.0:1784949154.230332:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 02000000:00000001:0.0:1784949154.230342:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949154.230343:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949154.230347:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949154.230350:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949154.230353:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949154.230356:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00010000:00000040:0.0:1784949154.230359:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677824 00010000:00000001:0.0:1784949154.230365:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:1.0:1784949154.230366:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003bfcf238 x1871651289677696/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:60/0 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:0.0:1784949154.230377:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000016685826 x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:60/0 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949154.230382:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949154.230392:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1784949154.230401:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003bfcf238 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289677696:12345-0@lo:400: Request processed in 727us (1420us total) trans 0 rc 0/0 00000400:00000001:2.0:1784949154.230405:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949154.230411:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:0.0:1784949154.230416:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000010:2.0:1784949154.230419:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00010000:00000001:0.0:1784949154.230424:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949154.230429:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000100:00100000:1.0:1784949154.230431:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 788 00000100:00001000:0.0:1784949154.230434:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=21 v=5 (1 1 1 1) 00000400:00000010:2.0:1784949154.230438:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000100:00000040:1.0:1784949154.230444:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949154.230450:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949154.230451:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949154.230452:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949154.230461:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:2.0:1784949154.230464:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949154.230465:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000010:1.0:1784949154.230471:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000329e608d. 00000100:00000001:0.0:1784949154.230476:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000010:1.0:1784949154.230484:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 02000000:00000001:0.0:1784949154.230488:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949154.230495:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949154.230497:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000100:00000001:0.0:1784949154.230507:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:2.0:1784949154.230508:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949154.230509:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:1.0:1784949154.230522:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000040:0.0:1784949154.230523:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000010:1.0:1784949154.230532:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000579609a4. 00000400:00000010:0.0:1784949154.230535:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 02000000:00000001:1.0:1784949154.230545:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949154.230547:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:0.0:1784949154.230548:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289677824, offset 224 00000100:00000001:1.0:1784949154.230555:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.230555:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949154.230567:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949154.230568:0:337119:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:1.0:1784949154.230570:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:2.0:1784949154.230577:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:1.0:1784949154.230581:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d6b8e52f. 00000100:00000001:2.0:1784949154.230581:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949154.230589:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949154.230590:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000010:1.0:1784949154.230595:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000042334b2c. 02000000:00000001:2.0:1784949154.230596:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949154.230608:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3693:3693:236:4294967295] 192.168.202.137@tcp LPNI seq info [2364:2364:8:4294967295] 00000100:00001000:2.0:1784949154.230609:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=26 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949154.230627:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=26 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949154.230636:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949154.230642:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949154.230648:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00010000:00000001:2.0:1784949154.230650:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949154.230656:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.230662:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949154.230669:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949154.230705:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005a1c05a3. 00000100:00000040:2.0:1784949154.230707:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784949154.230734:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949154.230746:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1784949154.230755:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:2.0:1784949154.230756:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949154.230762:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949154.230765:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005a1c05a3 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:2.0:1784949154.230773:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949154.230785:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000617daa1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289677696:0@lo:400: 00000100:00000001:2.0:1784949154.230793:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.230796:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949154.230798:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949154.230800:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000617daa1a x1871651289677696/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949154.230816:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949154.230820:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:0.0:1784949154.230829:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 02000000:00000001:2.0:1784949154.230830:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:0.0:1784949154.230839:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949154.230839:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000002fbb6924. 02000000:00000001:2.0:1784949154.230848:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949154.230850:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:2.0:1784949154.230853:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949154.230859:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949154.230863:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:0.0:1784949154.230866:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000016685826 x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:60/0 lens 224/224 e 0 to 0 dl 1784949165 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784949154.230869:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:2.0:1784949154.230877:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949154.230880:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949154.230885:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000617daa1a. 00000100:00000001:2.0:1784949154.230902:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949154.230948:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.230955:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949154.230958:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000016685826 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289677824:12345-192.168.202.137@tcp:400: Request processed in 706us (1184us total) trans 0 rc 0/0 00000100:00000001:2.0:1784949154.230964:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949154.230970:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.230977:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949154.230987:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949154.230995:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 764 00000100:00000040:0.0:1784949154.231006:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949154.231011:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949154.231017:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:3.0:1784949154.231023:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:0.0:1784949154.231024:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c343a817. 00000800:00000200:1.0:1784949154.231026:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1784949154.231037:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000800:00000010:1.0:1784949154.231038:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005a1c05a3. 00000800:00000001:3.0:1784949154.231046:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949154.231047:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000400:00000200:1.0:1784949154.231052:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:0.0:1784949154.231061:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000800:00000001:3.0:1784949154.231065:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949154.231069:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949154.231080:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949154.231080:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949154.231095:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:1.0:1784949154.231103:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000400:00000200:3.0:1784949154.231112:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949154.231116:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949154.231122:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949154.231136:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c000 00000400:00000200:3.0:1784949154.231147:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x906d [1] + 224 00000800:00000001:3.0:1784949154.231155:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949154.231175:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949154.231179:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949154.231183:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949154.231190:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949154.231194:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949154.231202:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949154.231224:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949154.231233:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949154.231236:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.231276:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949154.231280:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949154.231287:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949154.231289:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949154.231291:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000400:00000200:2.0:1784949154.231299:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6ceb0f1 00000400:00000010:2.0:1784949154.231301:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6ceb0f1. 00000100:00000001:2.0:1784949154.231304:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949154.231311:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949154.231323:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949154.231331:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949154.231333:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.231335:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949154.231337:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949154.231338:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949154.231340:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949154.231343:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949154.231349:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=21 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949154.231353:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=21 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949154.231356:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949154.231357:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949154.231359:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949154.231363:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949154.231372:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.231375:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949154.231377:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949154.231380:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949154.231382:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949154.231385:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949154.231393:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949154.231396:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949154.231397:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949154.231400:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949154.231408:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289677824:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949154.231413:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.231415:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949154.231417:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289677824/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949154.231425:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949154.231426:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949154.231428:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000378c7327. 02000000:00000001:2.0:1784949154.231432:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949154.231433:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949154.231434:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949154.231437:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949154.231438:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000820f02b3. 02000000:00000001:2.0:1784949154.231441:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949154.231442:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949154.231444:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:2.0:1784949154.231453:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949154.231455:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.231457:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949154.231461:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949154.231463:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949154.231465:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949154.231467:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949154.272011:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306093611, at 4306093612 00000100:00000010:1.0:1784949154.272028:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:2.0:1784949154.272052:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306093611, at 4306093612 00000100:00000040:0.0:1784949154.272055:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306093611, at 4306093612 00000100:00000010:2.0:1784949154.272070:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000029366a5. 00000100:00000040:3.0:1784949154.272071:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306093611, at 4306093612 00000100:00000010:0.0:1784949154.272076:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000010:3.0:1784949154.272091:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000068c34307. 00010000:00000001:2.0:1784949155.060056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949155.060065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949155.060073:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949155.060075:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949155.060141:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949155.060148:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949155.060154:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949155.252004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949155.252017:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949155.252020:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949155.252026:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949155.252030:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949155.252036:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949155.252040:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949156.084055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949156.084064:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949156.084071:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949156.084073:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949156.084196:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949156.084203:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949156.084208:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949156.276124:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949156.276141:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949156.276145:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949156.276152:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949156.276156:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949156.276161:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949156.276165:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1784949156.788061:0:344011:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 1.2.3.4:988 00000400:00000100:1.0:1784949156.788130:0:344011:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 1.2.3.4@tcp at host 1.2.3.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949156.788143:0:344011:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 1.2.3.4@tcp: down 00000400:00000200:1.0:1784949156.788151:0:344011:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784949156.788158:0:344011:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:1.0:1784949156.788163:0:344011:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000662152ce. 00000400:00000200:1.0:1784949156.788171:0:344011:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949156.788177:0:344011:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949156.788184:0:344011:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11429 health :900 00000400:00000200:1.0:1784949156.788191:0:344011:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949156.788196:0:344011:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:1.0:1784949156.788201:0:344011:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:0.0:1784949156.788330:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949156.788342:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:0.0:1784949156.788352:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949156.788355:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949156.788359:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:0.0:1784949156.788363:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000400:00000200:0.0:1784949156.788369:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949156.788374:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:0.0:1784949156.788379:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006620b0a1. 00000400:00000200:0.0:1784949156.788386:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:0.0:1784949156.788389:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:0.0:1784949156.788393:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00010000:00000001:2.0:1784949157.108042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949157.108051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949157.108058:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949157.108060:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949157.108179:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1784949157.108186:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1784949157.108191:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949157.300157:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949157.300172:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949157.300176:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949157.300182:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949157.300186:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949157.300190:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949157.300193:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949157.492102:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949157.492113:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000000286ff14. 00000400:00000010:3.0:1784949157.492127:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000378c7327. 00000400:00000010:3.0:1784949157.492135:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000400:00000010:3.0:1784949157.492147:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c986c2b1. 00000400:00000200:3.0:1784949157.492153:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949157.492176:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949157.492185:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949157.492191:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3694:3694:237:4294967295] 192.168.202.137@tcp LPNI seq info [2365:2365:8:4294967295] 00000400:00000200:3.0:1784949157.492202:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949157.492213:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949157.492220:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949157.492228:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000800:00000200:3.0:1784949157.492238:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949157.492250:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949157.492257:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949157.492285:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11431 health :900 00000800:00000200:2.0:1784949157.492513:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949157.492521:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000400:00000200:2.0:1784949157.492531:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949157.492542:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949157.492552:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949157.492556:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949157.492573:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949157.492585:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949157.492589:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949157.492594:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949157.492602:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949157.492621:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949157.492634:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949157.492684:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949157.492691:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3695:3695:237:4294967295] 192.168.202.137@tcp LPNI seq info [2366:2366:8:4294967295] 00000400:00000200:2.0:1784949157.492706:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949157.492713:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949157.492720:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ca1124f5. 00000800:00000200:2.0:1784949157.492728:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949157.492737:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949157.492742:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ca1124f5 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949157.492850:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949157.492854:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949157.492859:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ca1124f5. 00000400:00000200:2.0:1784949157.492866:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949157.492869:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949157.492875:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949157.492886:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949157.492898:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949157.492905:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949157.492957:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949157.492968:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784949157.492986:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9081 00000800:00000001:1.0:1784949157.492993:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949157.493003:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949157.493006:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949157.493010:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949157.493019:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:1.0:1784949157.493023:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000400:00000200:1.0:1784949157.493027:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949157.493032:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000000286ff14. 00000400:00000010:1.0:1784949157.493037:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000378c7327. 00000800:00000001:1.0:1784949157.493046:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949157.493049:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949158.132030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949158.132037:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949158.132044:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949158.132046:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949158.132171:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949158.132178:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949158.132183:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949158.324047:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949158.324054:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784949158.324069:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949133/real 1784949133] req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949158 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949158.324093:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949158.324096:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949158.324101:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fed0cd52. 00000400:00000200:1.0:1784949158.324108:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000400:00000010:1.0:1784949158.324112:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:1.0:1784949158.324118:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949158.324122:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949158 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949158.324133:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949158 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949158.324142:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949158.324144:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.324148:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949158.324151:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949158.324155:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949158 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949158.324164:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.324174:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949158.324176:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.324180:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949158.324183:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949158.324185:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949158.324189:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949158 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949158.324199:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949158.324203:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.324205:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949158.324206:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.324209:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784949158.324212:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784949158.324216:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784949158.324243:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:1.0:1784949158.324246:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784949158.324251:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949158 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949158.324260:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289676672:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784949158.324267:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.324270:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949158.324274:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289676672/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949158 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949158.324283:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949158.324287:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949158.324293:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000000239f355. 02000000:00000001:1.0:1784949158.324298:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949158.324300:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949158.324303:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:1.0:1784949158.324306:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949158.324309:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000001f4abcc4. 02000000:00000001:1.0:1784949158.324314:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949158.324317:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949158.324319:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 00000100:00000001:1.0:1784949158.324332:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949158.324334:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.324337:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.324341:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949158.324343:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.324346:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949158.324348:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949158.324358:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949158.324367:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949158.324375:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004ef1e045. 00000020:00000040:2.0:1784949158.324382:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949158.324388:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949158.324391:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949158.324397:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949158.324406:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ca1124f5. 00000100:00000001:2.0:1784949158.324414:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949158.324421:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949158.324436:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949158.324439:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949158.324445:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ef1e045] to pc [ptlrpcd_00_03+3] req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949158.324484:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949158.324488:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949158.324492:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784949158.324496:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1784949158.324499:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784949158.324504:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1784949158.324511:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11405 00000100:00000001:1.0:1784949158.324520:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949158.324523:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:1.0:1784949158.324526:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1784949158.324532:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11404 00000100:00000001:1.0:1784949158.324533:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949158.324546:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949158.324555:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000400:00000200:2.0:1784949158.324567:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:1.0:1784949158.324568:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949158.324575:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949158.324579:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949158.324586:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949158.324587:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949158.324592:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949158.324594:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:1.0:1784949158.324601:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949158.324603:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00100000:1.0:1784949158.324610:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004ef1e045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289677952:0@lo:400: 00000020:00000001:2.0:1784949158.324611:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949158.324617:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1784949158.324618:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:1.0:1784949158.324622:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1784949158.324635:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1784949158.324663:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1784949158.324668:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:3.0:1784949158.324668:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949158.324670:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949158.324684:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000001:2.0:1784949158.324687:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784949158.324690:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1784949158.324698:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000010:1.0:1784949158.324701:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004f865a97. 00000400:00000200:0.0:1784949158.324705:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 02000000:00000001:1.0:1784949158.324709:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949158.324714:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949158.324720:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949158.324723:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000010:0.0:1784949158.324724:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004b465f54. 00000400:00000010:1.0:1784949158.324725:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fab125ff. 00000100:00000001:2.0:1784949158.324726:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:2.0:1784949158.324737:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:0.0:1784949158.324739:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000400:00000010:1.0:1784949158.324743:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00080000:2.0:1784949158.324746:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00080000:2.0:1784949158.324755:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000010:0.0:1784949158.324756:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000034614846. 00000100:00000200:1.0:1784949158.324758:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289677952, portal 25 00000100:00000001:2.0:1784949158.324759:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949158.324769:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1784949158.324771:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1784949158.324772:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:1.0:1784949158.324779:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562019456 : -115544147532160 : ffff96e9c8216680) 02000000:00000001:2.0:1784949158.324780:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949158.324791:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949158.324797:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949158.324798:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:2.0:1784949158.324798:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1784949158.324805:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949158.324816:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:2.0:1784949158.324817:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fcdc5039. 00000020:00000040:2.0:1784949158.324829:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:0.0:1784949158.324830:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949158.324830:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949158.324841:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784949158.324843:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1784949158.324844:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000010:1.0:1784949158.324850:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 02000000:00000001:2.0:1784949158.324851:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949158.324855:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:2.0:1784949158.324865:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1784949158.324866:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289677952, offset 0 00000400:00000200:0.0:1784949158.324870:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3696:3696:237:4294967295] 1.2.3.4@tcp LPNI seq info [110:110:8:4294967295] 02000000:00000001:2.0:1784949158.324872:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949158.324879:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949158.324886:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784949158.324890:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 02000000:00000010:2.0:1784949158.324892:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006926284c. 00000100:00000001:2.0:1784949158.324901:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949158.324945:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:0.0:1784949158.324954:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000800:00000010:0.0:1784949158.324962:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002a753d9d. 00000400:00000200:1.0:1784949158.324971:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:2.0:1784949158.324972:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1784949158.324981:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.0:1784949158.324992:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000400:00000200:1.0:1784949158.325006:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c080 00000100:00000001:2.0:1784949158.325007:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949158.325015:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1784949158.325016:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784949158.325024:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1784949158.325027:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcdc5039] to pc [ptlrpcd_rcv+-1] req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949158.325028:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8685 [2] + 0 00000800:00000100:0.0:1784949158.325040:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000400:00000010:1.0:1784949158.325048:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000086a2279a. 00000800:00000010:0.0:1784949158.325050:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002a753d9d. 00000400:00000200:1.0:1784949158.325063:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b0990f86 00000400:00000200:0.0:1784949158.325065:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1784949158.325076:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949158.325079:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949158.325083:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000001:2.0:1784949158.325084:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949158.325094:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949158.325095:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000040:2.0:1784949158.325095:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949158.325110:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:0.0:1784949158.325111:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000010:2.0:1784949158.325120:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064fd9c10. 00000020:00000040:2.0:1784949158.325128:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949158.325134:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1784949158.325136:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0990f86 00000400:00000200:0.0:1784949158.325139:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 02000000:00000001:2.0:1784949158.325141:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1784949158.325142:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b0990f86. 00000100:00000001:3.0:1784949158.325147:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949158.325149:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.325155:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:0.0:1784949158.325161:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000200:1.0:1784949158.325164:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784949158.325165:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1784949158.325166:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b9b32818. 00000400:00000200:0.0:1784949158.325177:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000100:00000001:3.0:1784949158.325180:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949158.325182:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949158.325185:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000016fc8cf8 x1871651289677952 msgsize 224 00000400:00000001:0.0:1784949158.325193:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949158.325197:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949158.325204:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000100:00100000:1.0:1784949158.325205:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949158.325207:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1784949158.325214:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949158.325217:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000040:1.0:1784949158.325221:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784949158.325235:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:1.0:1784949158.325249:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949158.325259:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:0.0:1784949158.325279:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004b465f54. 00000400:00000200:0.0:1784949158.325305:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000400:00000200:1.0:1784949158.325306:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000100:00000001:2.0:1784949158.325320:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1784949158.325321:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000400:00000200:0.0:1784949158.325331:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:2.0:1784949158.325332:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949158.325338:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949158.325340:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949158.325352:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949158.325361:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000200:1.0:1784949158.325366:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949158.325367:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064fd9c10] to pc [ptlrpcd_00_00+0] req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949158.325370:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949158.325382:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949158.325404:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.325422:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949158.325431:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289677952 00000100:00100000:3.0:1784949158.325433:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fcdc5039 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289678080:1.2.3.4@tcp:8: 02000000:00000001:0.0:1784949158.325442:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949158.325452:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949158.325458:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949158.325466:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949158.325474:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949158.325476:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949158.325481:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949158.325487:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949158.325490:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289677952 00000100:00000001:2.0:1784949158.325492:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.325505:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949158.325508:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949158.325512:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:2.0:1784949158.325513:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11435) 00000020:00000001:0.0:1784949158.325528:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949158.325535:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949158.325538:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949158.325540:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949158.325550:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949158.325550:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949158.325551:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949158.325553:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784949158.325567:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1784949158.325572:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.325574:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949158.325575:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949158.325580:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949158.325586:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306097707 00000100:00000001:0.0:1784949158.325590:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949158.325593:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949158.325600:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000154da45a. 00000100:00000001:2.0:1784949158.325603:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949158.325603:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784949158.325604:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:0.0:1784949158.325614:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949158.325618:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949158.325618:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:0.0:1784949158.325623:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000154da45a. 00000100:00000001:1.0:1784949158.325627:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949158.325633:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949158.325638:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784949158.325675:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:3.0:1784949158.325676:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.325677:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306097707 00000100:00000001:2.0:1784949158.325677:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949158.325689:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000010:0.0:1784949158.325698:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c343a817. 00000100:00000010:1.0:1784949158.325704:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000010:0.0:1784949158.325708:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000040:2.0:1784949158.325712:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949158.325717:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1784949158.325728:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949158.325730:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949158.325732:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306097707 00000100:00000001:2.0:1784949158.325739:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.325741:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:1.0:1784949158.325743:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:3.0:1784949158.325746:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949158.325749:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:2.0:1784949158.325749:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949158.325752:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949158.325768:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949158.325769:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949158.325771:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949158.325772:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306097707 00000100:00000001:0.0:1784949158.325788:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949158.325793:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 02000000:00000001:2.0:1784949158.325795:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.325803:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00100000:2.0:1784949158.325804:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064fd9c10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289678208:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949158.325813:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949158.325816:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949158.325818:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949158.325821:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949158.325824:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949158.325829:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949158.325830:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.325841:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949158.325842:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:2.0:1784949158.325843:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949158.325850:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000010:2.0:1784949158.325856:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000002fbb6924. 00000100:00000001:1.0:1784949158.325860:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1784949158.325865:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 789 02000000:00000001:2.0:1784949158.325868:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.325869:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:0.0:1784949158.325873:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949158.325883:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742026320 : -115538967525296 : ffff96eafce20850) 00000100:00000001:1.0:1784949158.325884:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784949158.325884:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000100:00000001:1.0:1784949158.325890:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949158.325899:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949158.325902:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000040:0.0:1784949158.325947:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000016fc8cf8 x1871651289677952/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:64/0 lens 224/0 e 0 to 0 dl 1784949169 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949158.325948:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289678208, portal 10 00000100:00000001:1.0:1784949158.325951:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949158.325959:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949158.325965:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529562016768 : -115544147534848 : ffff96e9c8215c00) 00000100:00000001:0.0:1784949158.325981:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784949158.325984:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949158.325990:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000154da45a. 00000100:00000001:0.0:1784949158.325999:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949158.326009:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000154da45a. 00000100:00000001:2.0:1784949158.326013:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949158.326020:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000016fc8cf8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289677952:12345-0@lo:400: 00000100:00000040:2.0:1784949158.326026:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:0.0:1784949158.326036:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289677952 00000400:00000010:2.0:1784949158.326038:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000020:00000001:0.0:1784949158.326042:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000200:2.0:1784949158.326045:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289678208, offset 0 00000020:00000001:0.0:1784949158.326050:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949158.326056:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949158.326057:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949158.326065:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949158.326070:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949158.326074:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:2.0:1784949158.326076:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:0.0:1784949158.326078:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949158.326086:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:2.0:1784949158.326087:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1784949158.326091:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:2.0:1784949158.326097:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:0.0:1784949158.326099:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949158.326107:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1784949158.326108:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949158.326116:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:2.0:1784949158.326117:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:0.0:1784949158.326123:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:2.0:1784949158.326129:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3697:3697:237:4294967295] 192.168.202.137@tcp LPNI seq info [2367:2367:8:4294967295] 02000000:00000010:0.0:1784949158.326133:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000003a05a9. 02000000:00000001:0.0:1784949158.326140:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.326147:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.326155:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949158.326156:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00010000:00000040:0.0:1784949158.326165:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289677952 00000800:00000200:2.0:1784949158.326175:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00010000:00000001:0.0:1784949158.326176:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949158.326188:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000016fc8cf8 x1871651289677952/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:64/0 lens 224/224 e 0 to 0 dl 1784949169 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1784949158.326188:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000cee99234. 00000800:00000200:2.0:1784949158.326208:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00010000:00000001:0.0:1784949158.326212:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949158.326216:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949158.326221:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=30 v=5 (1 1 1 1) 00000800:00000200:2.0:1784949158.326223:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784949158.326233:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1784949158.326234:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000cee99234 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:0.0:1784949158.326241:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949158.326247:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949158.326251:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949158.326253:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.326256:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949158.326259:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949158.326262:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949158.326264:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000200:0.0:1784949158.326275:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289677952, offset 224 00000100:00000040:2.0:1784949158.326276:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2196 usecs (3 jiffies) 00000400:00000200:0.0:1784949158.326286:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949158.326287:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306097669 00000100:00000001:2.0:1784949158.326297:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306097707 00000100:00000001:2.0:1784949158.326307:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306097669 00000400:00000200:0.0:1784949158.326308:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949158.326319:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306097707 00000100:00000001:2.0:1784949158.326326:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306097669 00000100:00000001:2.0:1784949158.326330:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306097707 00000400:00000200:0.0:1784949158.326331:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c080 00000100:00000001:2.0:1784949158.326337:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306097669 00000400:00000200:0.0:1784949158.326345:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9089 [1] + 224 00000100:00000001:2.0:1784949158.326345:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306097707 00000100:00000001:2.0:1784949158.326353:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949158.326360:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949158.326362:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949158.326368:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949158.326377:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949158.326382:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.326385:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949158.326396:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949158.326408:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949158.326435:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949158.326443:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:0.0:1784949158.326445:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:0.0:1784949158.326449:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949158.326452:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949158.326457:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949158.326461:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784949158.326464:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949158.326466:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.326469:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949158.326473:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000016fc8cf8 x1871651289677952/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:64/0 lens 224/224 e 0 to 0 dl 1784949169 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949158.326486:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000016fc8cf8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289677952:12345-0@lo:400: Request processed in 503us (1298us total) trans 0 rc 0/0 00000800:00000200:3.0:1784949158.326492:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00100000:0.0:1784949158.326503:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 789 00000800:00000010:3.0:1784949158.326504:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000cee99234. 00000100:00000040:0.0:1784949158.326512:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949158.326518:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000200:3.0:1784949158.326519:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949158.326527:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949158.326536:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c343a817. 00000400:00000200:3.0:1784949158.326540:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:0.0:1784949158.326548:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949158.326555:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949158.326556:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000020:00000010:0.0:1784949158.326558:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000800:00000001:2.0:1784949158.326562:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949158.326564:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:3.0:1784949158.326578:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000020:00000040:0.0:1784949158.326583:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949158.326590:0:337114:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1784949158.326592:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000400:00000001:1.0:1784949158.326598:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:0.0:1784949158.326599:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000003a05a9. 00000800:00000001:2.0:1784949158.326600:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949158.326604:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949158.326606:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1784949158.326613:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000400:00000010:1.0:1784949158.326617:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fab125ff. 00000800:00000001:2.0:1784949158.326622:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949158.326624:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949158.326665:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000400:00000200:1.0:1784949158.326669:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000800:00000001:2.0:1784949158.326670:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949158.326681:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ee18875c. 00000400:00000010:1.0:1784949158.326685:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784949158.326706:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000010:0.0:1784949158.326709:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f0571d9e. 00000400:00000200:2.0:1784949158.326714:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:1.0:1784949158.326725:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949158.326744:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949158.326757:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c180 00000100:00000040:3.0:1784949158.326772:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949158.326774:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949158.326774:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 172400 00000800:00000001:2.0:1784949158.326815:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949158.326827:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949158.326843:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949158.326862:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949158.326870:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949158.326872:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949158.326877:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.326884:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:3.0:1784949158.326887:0:337114:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1784949158.326892:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949158.326892:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949158.326900:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784949158.326900:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:2.0:1784949158.326950:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000aaa80bc3. 00000100:00000040:2.0:1784949158.326957:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000aaa80bc3 x1871651289678208 msgsize 224 00000100:00000001:1.0:1784949158.326962:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949158.326970:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949158.326979:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00001000:1.0:1784949158.326981:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=30 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949158.326991:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=30 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949158.326996:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949158.326999:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949158.327001:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000001:1.0:1784949158.327003:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949158.327008:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.327012:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949158.327017:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949158.327020:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949158.327026:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949158.327038:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949158.327039:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784949158.327046:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289678208 00000100:00000001:1.0:1784949158.327047:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949158.327054:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949158.327055:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949158.327062:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:1.0:1784949158.327067:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949158.327068:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949158.327079:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949158.327087:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289678208 00000020:00000001:0.0:1784949158.327093:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00100000:1.0:1784949158.327098:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004ef1e045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289677952:0@lo:400: 00000020:00000040:0.0:1784949158.327099:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949158.327106:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949158.327110:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949158.327112:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:1.0:1784949158.327117:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784949158.327119:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:1.0:1784949158.327125:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004ef1e045 x1871651289677952/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949158.327127:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949158.327136:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949158.327144:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000154da45a. 00000100:00000001:1.0:1784949158.327148:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784949158.327150:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949158.327153:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:0.0:1784949158.327156:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000154da45a. 02000000:00000010:1.0:1784949158.327159:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004f865a97. 00000020:00000010:0.0:1784949158.327164:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 02000000:00000001:1.0:1784949158.327167:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:0.0:1784949158.327171:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c343a817. 00000020:00000001:1.0:1784949158.327173:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:0.0:1784949158.327178:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000020:00000040:1.0:1784949158.327179:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949158.327191:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:0.0:1784949158.327195:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000010:1.0:1784949158.327196:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ca1124f5. 00000100:00000001:0.0:1784949158.327204:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784949158.327206:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1784949158.327209:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:1.0:1784949158.327211:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1784949158.327215:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784949158.327216:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004ef1e045. 00000100:00000001:1.0:1784949158.327228:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949158.327229:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949158.327231:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949158.327235:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949158.327237:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949158.327241:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.327247:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949158.327252:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949158.327256:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949158.327259:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.327264:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949158.327280:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949158.327283:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949158.327291:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 765 00000100:00000040:0.0:1784949158.327295:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949158.327299:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562023040 : -115544147528576 : ffff96e9c8217480) 00000100:00000040:0.0:1784949158.327307:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000aaa80bc3 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:64/0 lens 224/0 e 0 to 0 dl 1784949169 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949158.327319:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949158.327322:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000154da45a. 00000100:00000001:0.0:1784949158.327326:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949158.327329:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000154da45a. 00000100:00100000:0.0:1784949158.327334:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000aaa80bc3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289678208:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949158.327339:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289678208 00000020:00000001:0.0:1784949158.327341:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949158.327344:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949158.327346:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.327348:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949158.327350:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949158.327354:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949158.327357:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949158.327359:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949158.327361:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949158.327363:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.327365:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949158.327370:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949158.327372:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949158.327375:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000003a05a9. 02000000:00000001:0.0:1784949158.327377:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.327379:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.327382:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949158.327384:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289678208 00010000:00000001:0.0:1784949158.327387:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949158.327391:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000aaa80bc3 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:64/0 lens 224/224 e 0 to 0 dl 1784949169 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949158.327400:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949158.327402:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949158.327406:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=25 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949158.327410:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949158.327413:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949158.327415:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949158.327418:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949158.327419:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949158.327422:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949158.327425:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949158.327427:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000200:0.0:1784949158.327431:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289678208, offset 224 00000400:00000200:0.0:1784949158.327436:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949158.327445:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949158.327452:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3698:3698:237:4294967295] 192.168.202.137@tcp LPNI seq info [2368:2368:8:4294967295] 00000400:00000200:0.0:1784949158.327464:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949158.327470:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949158.327474:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002a753d9d. 00000800:00000200:0.0:1784949158.327479:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949158.327486:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949158.327489:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002a753d9d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949158.327507:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949158.327511:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949158.327513:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949158.327515:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949158.327517:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949158.327521:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000aaa80bc3 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:64/0 lens 224/224 e 0 to 0 dl 1784949169 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949158.327532:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000aaa80bc3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289678208:12345-192.168.202.137@tcp:400: Request processed in 212us (574us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949158.327542:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 765 00000100:00000040:0.0:1784949158.327545:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949158.327549:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949158.327552:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949158.327556:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c343a817. 00000020:00000010:0.0:1784949158.327560:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949158.327564:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000020:00000040:0.0:1784949158.327569:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949158.327572:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949158.327743:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949158.327749:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002a753d9d. 00000400:00000200:1.0:1784949158.327754:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949158.327761:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949158.327766:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:1.0:1784949158.327768:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000100:00000001:1.0:1784949158.327772:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949158.327774:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949158.327781:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949158.327788:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949158.327791:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949158.327794:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949158.327801:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949158.327813:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c180 00000400:00000200:1.0:1784949158.327821:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9091 [1] + 224 00000800:00000001:1.0:1784949158.327828:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949158.327840:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949158.327842:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949158.327850:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949158.327856:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949158.327862:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949158.327874:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949158.327937:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949158.327945:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949158.327949:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949158.328013:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949158.328019:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949158.328026:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949158.328028:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949158.328031:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000400:00000200:2.0:1784949158.328039:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:2.0:1784949158.328041:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000100:00000001:2.0:1784949158.328045:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949158.328052:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949158.328065:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949158.328076:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949158.328077:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949158.328080:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949158.328082:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949158.328084:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949158.328086:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949158.328090:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949158.328097:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=25 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949158.328100:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=25 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949158.328103:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949158.328105:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949158.328107:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949158.328111:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949158.328120:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949158.328124:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949158.328126:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949158.328128:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949158.328130:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949158.328134:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949158.328142:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949158.328144:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949158.328146:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949158.328149:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949158.328157:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064fd9c10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289678208:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949158.328162:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949158.328165:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949158.328167:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064fd9c10 x1871651289678208/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949174 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949158.328175:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949158.328176:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949158.328179:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000002fbb6924. 02000000:00000001:2.0:1784949158.328182:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949158.328184:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949158.328185:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949158.328188:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949158.328191:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b9b32818. 02000000:00000001:2.0:1784949158.328194:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949158.328196:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949158.328198:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064fd9c10. 00000100:00000001:2.0:1784949158.328201:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949158.328203:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949158.328205:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949158.328210:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949158.328212:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949158.328215:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949158.328218:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949158.368029:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306097707, at 4306097708 00000100:00000010:3.0:1784949158.368053:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:0.0:1784949158.368062:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306097707, at 4306097708 00000100:00000010:0.0:1784949158.368078:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:1.0:1784949158.368079:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306097707, at 4306097708 00000100:00000040:2.0:1784949158.368081:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306097707, at 4306097708 00000100:00000010:1.0:1784949158.368097:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:2.0:1784949158.368102:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784949159.156127:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949159.156138:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949159.156146:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949159.156149:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949159.156240:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949159.156251:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949159.156257:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949159.348090:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949159.348097:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1784949159.348115:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949158/real 0] req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949159 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.348134:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.348138:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949159.348141:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949159.348145:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949159 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.348156:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.348166:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949159.348169:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.348172:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949159.348176:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949159.348179:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949159.348184:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949159 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.348193:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1784949159.348198:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.348200:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949159.348202:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.348205:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1784949159.348208:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1784949159.348212:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1784949159.348246:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:3.0:1784949159.348250:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1784949159.348255:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949159 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949159.348268:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fcdc5039 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289678080:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784949159.348275:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.348282:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949159.348286:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fcdc5039 x1871651289678080/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949159 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.348296:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1784949159.348302:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949159.348304:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:3.0:1784949159.348309:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:0.0:1784949159.348310:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:3.0:1784949159.348315:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000006926284c. 00000100:00000001:0.0:1784949159.348333:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:3.0:1784949159.348334:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949159.348345:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949159.348347:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fcdc5039. 00000100:00000010:0.0:1784949159.348354:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000100:00000001:3.0:1784949159.348355:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949159.348360:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.348369:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949159.348371:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949159.348375:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949159.348379:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.348380:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784949159.348386:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949159.348387:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949159.348392:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949159.348399:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949159.348414:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002a753d9d. 00000100:00000001:0.0:1784949159.348426:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949159.348437:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949159.348455:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949159.348458:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949159.348464:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_01+1] req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949159.348502:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949159.348506:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949159.348510:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949159.348514:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949159.348516:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949159.348522:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949159.348529:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11405 00000400:00000200:0.0:1784949159.348541:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784949159.348548:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11430 00000100:00000001:3.0:1784949159.348560:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949159.348568:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:3.0:1784949159.348573:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949159.348577:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:3.0:1784949159.348584:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1784949159.348585:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:3.0:1784949159.348589:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1784949159.348592:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1784949159.348596:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949159.348602:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949159.348603:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949159.348608:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949159.348613:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949159.348625:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949159.348626:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949159.348655:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949159.348670:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949159.348683:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784949159.348701:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949159.348713:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949159.348714:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1784949159.348715:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000040:0.0:1784949159.348721:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784949159.348730:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1784949159.348731:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1784949159.348736:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000200:1.0:1784949159.348739:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 02000000:00000001:3.0:1784949159.348741:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949159.348742:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:1.0:1784949159.348754:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004f865a97. 00000020:00000001:0.0:1784949159.348755:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784949159.348755:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949159.348775:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00100000:3.0:1784949159.348777:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289678336:0@lo:400: 00000100:00000040:0.0:1784949159.348778:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000400:00000010:1.0:1784949159.348792:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003859b1c5. 00000100:00000001:0.0:1784949159.348793:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949159.348794:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949159.348801:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949159.348805:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000200:1.0:1784949159.348808:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 02000000:00000001:3.0:1784949159.348809:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1784949159.348811:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:3.0:1784949159.348822:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949159.348828:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 02000000:00000001:3.0:1784949159.348829:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949159.348837:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00080000:0.0:1784949159.348841:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 02000000:00000010:3.0:1784949159.348842:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a17860fa. 02000000:00000001:3.0:1784949159.348856:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.348858:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949159.348858:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784949159.348877:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1784949159.348877:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000010:3.0:1784949159.348882:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002fbcdf42. 00000400:00000200:1.0:1784949159.348886:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784949159.348887:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949159.348897:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949159.348902:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000100:00000001:0.0:1784949159.348903:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1784949159.348903:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:0.0:1784949159.348908:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949159.348909:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3699:3699:237:4294967295] 1.2.3.4@tcp LPNI seq info [111:111:8:4294967295] 00000100:00000010:0.0:1784949159.348916:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000400:00000200:1.0:1784949159.348949:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000020:00000040:0.0:1784949159.348951:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:3.0:1784949159.348955:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289678336, portal 25 02000000:00000001:0.0:1784949159.348958:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949159.348963:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949159.348971:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:3.0:1784949159.348973:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949159.348974:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784949159.348982:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 02000000:00000001:0.0:1784949159.348983:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.0:1784949159.348984:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 02000000:00000001:0.0:1784949159.348990:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949159.349000:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005c010cf6. 02000000:00000010:0.0:1784949159.349003:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078a163a8. 00000100:00000040:3.0:1784949159.349010:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949159.349014:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949159.349015:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00100000:0.0:1784949159.349027:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1784949159.349031:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1784949159.349058:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:1.1:1784949159.349075:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 00000800:00000100:1.0:1784949159.349104:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000800:00000010:1.0:1784949159.349114:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005c010cf6. 00000100:00000001:3.0:1784949159.349116:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949159.349123:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949159.349140:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1784949159.349141:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000400:00000200:1.0:1784949159.349148:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1784949159.349149:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949159.349152:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289678336, offset 0 00000100:00000040:0.0:1784949159.349167:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_rcv+-1] req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949159.349176:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949159.349182:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949159.349228:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949159.349228:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949159.349236:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949159.349249:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000040:0.0:1784949159.349253:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949159.349261:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949159.349268:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000400:00000200:1.0:1784949159.349277:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:3.0:1784949159.349282:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c200 00000020:00000040:0.0:1784949159.349285:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949159.349292:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784949159.349307:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:0.0:1784949159.349309:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949159.349312:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949159.349316:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949159.349320:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1784949159.349321:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 02000000:00000001:0.0:1784949159.349324:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949159.349331:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x869d [2] + 0 02000000:00000010:0.0:1784949159.349343:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000662152ce. 00000100:00000040:2.0:1784949159.349344:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949159.349349:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000010:3.0:1784949159.349360:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000529cd956. 00000100:00000001:0.0:1784949159.349362:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949159.349384:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c48eba80 00000100:00000040:0.0:1784949159.349390:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949159.349392:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949159.349402:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949159.349404:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:3.0:1784949159.349411:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000001:1.0:1784949159.349413:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1784949159.349415:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784949159.349422:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 02000000:00000001:2.0:1784949159.349428:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784949159.349434:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:0.0:1784949159.349438:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1784949159.349445:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:2.0:1784949159.349452:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.349454:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949159.349455:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949159.349461:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c48eba80 00000100:00100000:2.0:1784949159.349466:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289678464:192.168.202.137@tcp:8: 00000400:00000010:3.0:1784949159.349470:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000c48eba80. 00000100:00000040:0.0:1784949159.349485:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_02+2] req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949159.349485:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:2.0:1784949159.349487:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949159.349491:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949159.349501:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:1.0:1784949159.349503:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004f865a97. 02000000:00000001:2.0:1784949159.349503:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949159.349517:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949159.349527:0:337119:0:(events.c:337:request_in_callback()) incoming req@0000000038d13d38 x1871651289678336 msgsize 224 02000000:00000001:2.0:1784949159.349530:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949159.349539:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 02000000:00000010:2.0:1784949159.349547:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006926284c. 00000100:00000001:0.0:1784949159.349555:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949159.349556:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 02000000:00000001:2.0:1784949159.349557:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949159.349559:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949159.349581:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11436) 00000100:00000040:3.0:1784949159.349585:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784949159.349589:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000400:00000010:2.0:1784949159.349590:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000400:00000010:2.0:1784949159.349605:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000001:0.0:1784949159.349613:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784949159.349618:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289678464, portal 4 00000100:00000001:3.0:1784949159.349621:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949159.349622:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949159.349626:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949159.349629:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949159.349659:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949159.349660:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 00000100:00000001:1.0:1784949159.349663:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949159.349673:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949159.349676:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0e1f1a0 00000400:00000010:3.0:1784949159.349693:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0e1f1a0. 00000100:00000040:2.0:1784949159.349706:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.349708:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949159.349725:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949159.349727:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784949159.349735:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1784949159.349736:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289678336 00000100:00000001:0.0:1784949159.349744:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949159.349746:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949159.349755:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949159.349756:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949159.349762:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949159.349764:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949159.349780:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949159.349781:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949159.349783:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949159.349799:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289678336 00000100:00100000:0.0:1784949159.349804:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289678592:192.168.202.137@tcp:400: 00000100:00000040:2.0:1784949159.349804:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949159.349806:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949159.349808:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1784949159.349815:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949159.349822:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949159.349823:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:2.0:1784949159.349823:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000100:00000040:3.0:1784949159.349829:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949159.349831:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 02000000:00000001:0.0:1784949159.349833:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000200:2.0:1784949159.349835:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289678464, offset 0 02000000:00000001:0.0:1784949159.349841:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949159.349844:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:0.0:1784949159.349857:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1784949159.349861:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:2.0:1784949159.349870:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784949159.349872:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 00000100:00000001:1.0:1784949159.349876:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:0.0:1784949159.349889:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949159.349894:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000556e55b. 00000100:00000001:1.0:1784949159.349906:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784949159.349907:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007679eba0. 00000020:00000010:1.0:1784949159.349909:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000556e55b. 00000400:00000010:0.0:1784949159.349916:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 00000400:00000200:2.0:1784949159.349928:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949159.349936:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949159.349941:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949159.349944:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949159.349949:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:1.0:1784949159.349950:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00000100:00000200:0.0:1784949159.349951:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289678592, portal 10 00000100:00000001:3.0:1784949159.349958:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949159.349959:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3700:3700:237:4294967295] 192.168.202.137@tcp LPNI seq info [2369:2369:8:4294967295] 00000100:00000001:0.0:1784949159.349960:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000010:1.0:1784949159.349964:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002afdbc54. 00000100:00000001:0.0:1784949159.349965:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000100:00000001:3.0:1784949159.349975:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1784949159.349977:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:0.0:1784949159.349983:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949159.349990:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:2.0:1784949159.349996:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949159.349997:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949159.350001:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949159.350005:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350006:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000200:2.0:1784949159.350012:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949159.350016:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949159.350017:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949159.350020:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b9b32818. 00000100:00000001:3.0:1784949159.350020:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949159.350023:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000089c8262f. 00000100:00000001:1.0:1784949159.350030:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949159.350034:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289678592, offset 0 00000800:00000200:2.0:1784949159.350044:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949159.350045:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306098732 00000400:00000200:0.0:1784949159.350046:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949159.350049:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949159.350065:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000010:3.0:1784949159.350073:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000068c34307. 00000400:00000200:0.0:1784949159.350075:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000800:00000200:2.0:1784949159.350075:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:1.0:1784949159.350077:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 790 00000800:00000200:2.0:1784949159.350087:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b9b32818 type 1, nob 616 niov 1 nkiov 1 00000100:00000040:1.0:1784949159.350090:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949159.350095:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000200:0.0:1784949159.350096:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949159.350096:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742024272 : -115538967527344 : ffff96eafce20050) 00000100:00000001:2.0:1784949159.350104:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949159.350118:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:2.0:1784949159.350118:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000040:1.0:1784949159.350120:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000038d13d38 x1871651289678336/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:65/0 lens 224/0 e 0 to 0 dl 1784949170 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.350121:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306098732 00000400:00000200:0.0:1784949159.350131:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784949159.350134:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306098732 00000100:00000010:3.0:1784949159.350135:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000400:00000200:0.0:1784949159.350145:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:2.0:1784949159.350157:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306098732 00000100:00000001:3.0:1784949159.350160:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949159.350161:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949159.350163:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3701:3701:236:4294967295] 192.168.202.137@tcp LPNI seq info [2370:2370:7:4294967295] 00000100:00000001:2.0:1784949159.350165:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306098732 00000020:00000010:1.0:1784949159.350171:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000556e55b. 00000100:00000001:3.0:1784949159.350183:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306098732 00000100:00000001:1.0:1784949159.350185:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949159.350185:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306098732 00000400:00000200:0.0:1784949159.350198:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:1.0:1784949159.350203:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000556e55b. 00000100:00000001:2.0:1784949159.350205:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306098732 00000100:00000010:3.0:1784949159.350206:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00100000:1.0:1784949159.350221:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000038d13d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289678336:12345-0@lo:400: 00000800:00000200:0.0:1784949159.350222:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949159.350226:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000800:00000010:0.0:1784949159.350231:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000820f02b3. 00000100:00000001:3.0:1784949159.350250:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306098732 00000100:00000001:2.0:1784949159.350253:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306098732 00000100:00000200:1.0:1784949159.350266:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289678336 00000100:00000010:3.0:1784949159.350270:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000029366a5. 00000100:00000001:2.0:1784949159.350272:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306098732 00000800:00000200:0.0:1784949159.350278:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:1.0:1784949159.350280:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949159.350288:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000001:1.0:1784949159.350294:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949159.350301:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.350304:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949159.350305:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306098732 00000800:00000200:0.0:1784949159.350311:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000020:00000001:1.0:1784949159.350314:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949159.350325:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784949159.350328:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1784949159.350337:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000820f02b3 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949159.350337:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949159.350340:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949159.350341:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949159.350350:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949159.350366:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.350370:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949159.350371:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949159.350372:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949159.350383:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949159.350390:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949159.350392:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949159.350392:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350395:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949159.350404:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:1.0:1784949159.350416:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:0.0:1784949159.350418:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2197 usecs (3 jiffies) 00000100:00000001:2.0:1784949159.350421:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949159.350427:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784949159.350429:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949159.350431:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306098693 02000000:00000001:1.0:1784949159.350433:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949159.350441:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350443:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306098732 02000000:00000010:1.0:1784949159.350446:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001f4abcc4. 00000100:00000001:2.0:1784949159.350448:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350451:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306098693 02000000:00000001:1.0:1784949159.350459:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350460:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306098732 00000100:00000001:0.0:1784949159.350469:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306098693 00000100:00000001:1.0:1784949159.350470:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350473:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306098732 00000020:00000001:1.0:1784949159.350477:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350483:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306098693 00010000:00000040:1.0:1784949159.350485:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289678336 00000100:00000001:0.0:1784949159.350490:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306098732 00010000:00000001:1.0:1784949159.350492:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949159.350495:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949159.350501:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:1.0:1784949159.350502:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000038d13d38 x1871651289678336/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:65/0 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949159.350508:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350516:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949159.350528:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949159.350529:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1784949159.350531:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350536:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00001000:1.0:1784949159.350540:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=31 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949159.350542:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949159.350548:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949159.350553:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949159.350555:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949159.350558:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949159.350558:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:1.0:1784949159.350566:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949159.350582:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000010:3.0:1784949159.350583:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b9b32818. 02000000:00000001:1.0:1784949159.350602:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:3.0:1784949159.350605:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1784949159.350613:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949159.350654:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949159.350658:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949159.350680:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000800:00000001:0.0:1784949159.350687:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784949159.350688:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000100:00000040:1.0:1784949159.350689:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000200:2.0:1784949159.350698:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000010:1.0:1784949159.350705:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000800:00000001:0.0:1784949159.350706:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949159.350709:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000820f02b3. 00000100:00000001:3.0:1784949159.350711:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949159.350712:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289678336, offset 224 00000800:00000001:0.0:1784949159.350713:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949159.350726:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949159.350729:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949159.350733:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949159.350737:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949159.350752:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949159.350763:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949159.350770:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000400:00000200:1.0:1784949159.350773:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:2.0:1784949159.350782:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000100:00000001:2.0:1784949159.350797:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784949159.350809:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c300 00000100:00000200:2.0:1784949159.350814:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949159.350815:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c200 00000100:00000001:3.0:1784949159.350829:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949159.350841:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 172624 00000100:00000040:3.0:1784949159.350845:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949184 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949159.350864:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949159.350875:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x90a9 [1] + 224 00000800:00000001:0.0:1784949159.350881:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949159.350895:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949159.350898:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949159.350907:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949159.350911:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949159.350937:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949159.350947:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949159.350954:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949159.350959:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949159.350964:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.350967:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949159.350976:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784949159.350978:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:0.0:1784949159.350988:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000617daa1a. 00000800:00000001:2.0:1784949159.350991:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949159.351001:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000617daa1a x1871651289678592 msgsize 224 00000800:00000001:2.0:1784949159.351002:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.351010:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949159.351012:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949159.351018:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949159.351019:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.351025:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1784949159.351032:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784949159.351042:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949159.351048:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949159.351058:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643c280 00000100:00000001:1.0:1784949159.351060:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949159.351061:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949159.351071:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949159.351077:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000100:2.0:1784949159.351077:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289678464 offset 0 length 520: 4 00000400:00000010:1.0:1784949159.351084:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000400:00000200:2.0:1784949159.351085:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 0000000008ad44ac is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:3.0:1784949159.351091:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784949159.351092:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:1.0:1784949159.351093:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00100000:3.0:1784949159.351106:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289678592 00000100:00000001:1.0:1784949159.351108:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949159.351110:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949159.351111:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949159.351124:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949159.351125:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949159.351125:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949159.351125:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.351148:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949159.351159:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949159.351164:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:0.0:1784949159.351165:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1784949159.351165:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:0.0:1784949159.351175:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1784949159.351176:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949159.351179:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000800:00000001:2.0:1784949159.351180:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949159.351181:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002fbcdf42. 00000020:00000001:1.0:1784949159.351191:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949159.351203:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000020:00000001:1.0:1784949159.351205:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00100000:3.0:1784949159.351208:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289678592 00000400:00000010:0.0:1784949159.351210:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000100:00000040:1.0:1784949159.351222:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000038d13d38 x1871651289678336/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:65/0 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949159.351223:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949159.351225:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:3.0:1784949159.351236:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000200:0.0:1784949159.351247:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949159.351251:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949159.351263:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949159.351283:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949159.351301:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00100000:1.0:1784949159.351305:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000038d13d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289678336:12345-0@lo:400: Request processed in 1131us (1765us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949159.351320:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949159.351334:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000200:0.0:1784949159.351341:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.351350:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1784949159.351351:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 790 00000020:00000010:3.0:1784949159.351375:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000100:00000040:1.0:1784949159.351377:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949159.351394:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949159.351398:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949159.351402:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949159.351404:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949159.351405:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000043d62367. 00000100:00000001:0.0:1784949159.351409:0:337119:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:1.0:1784949159.351415:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002afdbc54. 02000000:00000001:0.0:1784949159.351426:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:1.0:1784949159.351432:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:3.0:1784949159.351433:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ff5742ee. 00000100:00000001:0.0:1784949159.351434:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949159.351445:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949159.351454:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949159.351457:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000020:00000010:1.0:1784949159.351458:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000100:00001000:0.0:1784949159.351465:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=31 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949159.351486:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=31 v=5 (1 1 1 1) 00000020:00000040:1.0:1784949159.351489:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784949159.351495:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000040:3.0:1784949159.351496:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:1.0:1784949159.351498:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949159.351509:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000001f4abcc4. 00000100:00000001:0.0:1784949159.351511:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.351512:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784949159.351517:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949159.351525:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949159.351528:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784949159.351530:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784949159.351532:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949159.351539:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:0.0:1784949159.351540:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.351547:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.351549:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000010:1.0:1784949159.351551:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fab125ff. 00000100:00000040:0.0:1784949159.351556:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.351558:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000010:1.0:1784949159.351566:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d1142873. 00000100:00000001:3.0:1784949159.351568:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949159.351590:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949159.351594:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949159.351597:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949159.351601:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.351610:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949159.351614:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289678336:0@lo:400: 00000100:00000001:0.0:1784949159.351654:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949159.351658:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949159.351662:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949159.351663:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289678336/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.351671:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949159.351688:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1784949159.351688:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 766 00000100:00000040:3.0:1784949159.351708:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:0.0:1784949159.351711:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1784949159.351714:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554966272 : -115544154585344 : ffff96e9c7b5c700) 02000000:00000010:0.0:1784949159.351716:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a17860fa. 02000000:00000001:0.0:1784949159.351734:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949159.351742:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000617daa1a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:65/0 lens 224/0 e 0 to 0 dl 1784949170 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949159.351744:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949159.351749:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949159.351762:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949159.351769:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002a753d9d. 02000000:00000001:0.0:1784949159.351784:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1784949159.351787:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:0.0:1784949159.351789:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949159.351793:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000020:00000010:3.0:1784949159.351794:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:3.0:1784949159.351800:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949159.351803:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000100:00000001:0.0:1784949159.351806:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949159.351810:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949159.351814:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000617daa1a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289678592:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949159.351816:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949159.351833:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289678592 00000100:00000001:0.0:1784949159.351834:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949159.351844:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949159.351844:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949159.351857:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949159.351858:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949159.351866:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949159.351866:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949159.351884:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949159.351890:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949159.351896:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949159.351900:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949159.351942:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949159.351947:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949159.351951:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949159.351956:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949159.351963:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949159.351967:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949159.351974:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009f7bccfc. 02000000:00000001:3.0:1784949159.351980:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.351985:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949159.351990:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949159.351995:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289678592 00010000:00000001:3.0:1784949159.352000:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949159.352008:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000617daa1a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:65/0 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949159.352033:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949159.352039:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949159.352045:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=26 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949159.352051:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949159.352055:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949159.352058:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949159.352062:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949159.352065:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.352069:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949159.352072:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949159.352077:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000100:00000200:3.0:1784949159.352082:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289678592, offset 224 00000400:00000200:3.0:1784949159.352088:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949159.352101:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949159.352110:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3702:3702:237:4294967295] 192.168.202.137@tcp LPNI seq info [2371:2371:8:4294967295] 00000400:00000200:3.0:1784949159.352125:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949159.352132:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949159.352138:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784949159.352146:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949159.352156:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949159.352160:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949159.352186:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949159.352190:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949159.352193:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949159.352195:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949159.352198:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949159.352204:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000617daa1a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:65/0 lens 224/224 e 0 to 0 dl 1784949170 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949159.352216:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000617daa1a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289678592:12345-192.168.202.137@tcp:400: Request processed in 429us (1213us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949159.352227:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 766 00000100:00000040:3.0:1784949159.352232:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949159.352235:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949159.352237:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949159.352243:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ff5742ee. 00000020:00000010:3.0:1784949159.352249:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949159.352254:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000043d62367. 00000020:00000040:3.0:1784949159.352261:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949159.352265:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949159.352418:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949159.352426:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:0.0:1784949159.352432:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949159.352440:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949159.352446:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0e1f1a0 00000400:00000010:0.0:1784949159.352449:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0e1f1a0. 00000100:00000001:0.0:1784949159.352453:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949159.352455:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949159.352464:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949159.352471:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949159.352475:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949159.352478:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949159.352487:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949159.352500:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c300 00000400:00000200:0.0:1784949159.352507:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x90b9 [1] + 224 00000800:00000001:0.0:1784949159.352515:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949159.352524:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949159.352526:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949159.352530:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949159.352534:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949159.352540:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949159.352552:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949159.352576:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949159.352582:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949159.352585:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.352689:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949159.352694:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949159.352702:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949159.352704:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949159.352708:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007679eba0. 00000400:00000200:3.0:1784949159.352714:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000400:00000010:3.0:1784949159.352717:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000100:00000001:3.0:1784949159.352720:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949159.352728:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949159.352740:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.352752:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949159.352755:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.352759:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949159.352763:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949159.352766:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949159.352770:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949159.352774:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949159.352785:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=26 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949159.352792:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=26 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949159.352797:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949159.352800:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949159.352805:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949159.352813:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949159.352827:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.352833:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949159.352835:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949159.352840:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949159.352843:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949159.352849:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.352863:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949159.352867:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949159.352870:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949159.352876:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949159.352891:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289678592:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949159.352899:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.352902:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949159.352944:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289678592/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949159.352959:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949159.352962:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949159.352967:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 02000000:00000001:3.0:1784949159.352977:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949159.352980:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949159.352983:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949159.352987:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949159.352991:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000662152ce. 02000000:00000001:3.0:1784949159.352996:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949159.352999:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949159.353002:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:3.0:1784949159.353016:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949159.353018:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.353022:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949159.353030:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949159.353033:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949159.353036:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949159.353039:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949159.393018:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306098732, at 4306098733 00000100:00000040:1.0:1784949159.393023:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306098732, at 4306098733 00000100:00000010:2.0:1784949159.393037:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000010:1.0:1784949159.393041:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000040:0.0:1784949159.393063:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306098732, at 4306098733 00000100:00000010:0.0:1784949159.393082:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000068c34307. 00000100:00000040:3.0:1784949159.393086:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306098732, at 4306098733 00000100:00000010:3.0:1784949159.393101:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000029366a5. 00010000:00000001:2.0:1784949160.180018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949160.180027:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949160.180036:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949160.180038:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949160.180167:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949160.180175:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949160.180179:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1784949160.884141:0:343958:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.15:988 00000400:00000100:1.0:1784949160.884160:0:344054:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.11:988 00000400:00000100:3.0:1784949160.884209:0:343954:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.28:988 00000400:00000100:0.0:1784949160.884231:0:344056:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.9:988 00000400:00000100:2.0:1784949160.884257:0:343958:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.15@tcp at host 111.222.173.15:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784949160.884274:0:344054:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.11@tcp at host 111.222.173.11:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949160.884285:0:343958:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.15@tcp: down 00000400:00000200:2.0:1784949160.884300:0:343958:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949160.884311:0:344054:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.11@tcp: down 00000400:00000200:1.0:1784949160.884326:0:344054:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1784949160.884326:0:343958:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:2.0:1784949160.884339:0:343958:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000800:00000100:1.0:1784949160.884355:0:344054:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000400:00000200:2.0:1784949160.884357:0:343958:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000010:1.0:1784949160.884365:0:344054:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:2.0:1784949160.884378:0:343958:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949160.884380:0:344054:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000100:0.0:1784949160.884389:0:344056:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.9@tcp at host 111.222.173.9:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784949160.884400:0:343954:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.28@tcp at host 111.222.173.28:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949160.884401:0:343958:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11433 health :900 00000400:00000200:1.0:1784949160.884410:0:344054:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949160.884428:0:343958:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949160.884429:0:344056:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.9@tcp: down 00000400:00000200:1.0:1784949160.884432:0:344054:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949160.884442:0:343954:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.28@tcp: down 00000400:00000200:2.0:1784949160.884452:0:343958:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:0.0:1784949160.884453:0:344056:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949160.884454:0:344054:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000800:00000200:3.0:1784949160.884476:0:343954:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.173.28@tcp down 00000400:00000200:2.0:1784949160.884478:0:343958:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000100:0.0:1784949160.884495:0:344053:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.7:988 00000400:00000200:1.0:1784949160.884513:0:344054:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000100:0.0:1784949160.884546:0:344053:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.7@tcp at host 111.222.173.7:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949160.884558:0:344053:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.7@tcp: down 00000400:00000200:0.0:1784949160.884567:0:344053:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1784949160.884583:0:344053:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000400:00000100:3.0:1784949160.884586:0:344052:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.13:988 00000800:00000010:0.0:1784949160.884589:0:344053:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fd30a9ad. 00000400:00000100:2.0:1784949160.884593:0:344046:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.1:988 00000400:00000200:1.0:1784949160.884606:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949160.884609:0:344053:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949160.884651:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000200:0.0:1784949160.884657:0:344053:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000001:1.0:1784949160.884669:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949160.884677:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949160.884683:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:1.0:1784949160.884689:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:1.0:1784949160.884702:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000100:2.0:1784949160.884703:0:344046:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.1@tcp at host 111.222.173.1:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949160.884704:0:344053:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949160.884718:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:0.0:1784949160.884726:0:344053:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000100:3.0:1784949160.884726:0:344052:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.13@tcp at host 111.222.173.13:988 took too long: that node may be hung or experiencing high load. 00000400:00000010:1.0:1784949160.884733:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000047e19e91. 00000400:00000200:2.0:1784949160.884734:0:344046:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.1@tcp: down 00000400:00000200:2.0:1784949160.884757:0:344046:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949160.884759:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:3.0:1784949160.884766:0:344052:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.13@tcp: down 00000400:00000200:0.0:1784949160.884782:0:344053:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:1.0:1784949160.884785:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000800:00000100:2.0:1784949160.884786:0:344046:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000400:00000200:3.0:1784949160.884797:0:344052:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:2.0:1784949160.884810:0:344046:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fb9675a2. 00000400:00000200:1.0:1784949160.884819:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000400:00000200:1.0:1784949160.884832:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000200:2.0:1784949160.884836:0:344046:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000001:1.0:1784949160.884837:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949160.884843:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000100:0.0:1784949160.884847:0:344055:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.5:988 00000400:00000200:1.0:1784949160.884849:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:1.0:1784949160.884858:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:2.0:1784949160.884863:0:344046:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949160.884871:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000100:3.0:1784949160.884880:0:344057:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.3:988 00000400:00000200:2.0:1784949160.884883:0:344046:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949160.884889:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000100:0.0:1784949160.884902:0:344055:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.5@tcp at host 111.222.173.5:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949160.884990:0:344055:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.5@tcp: down 00000400:00000200:2.0:1784949160.885002:0:344046:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000010:1.0:1784949160.885005:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000da755428. 00000400:00000200:0.0:1784949160.885019:0:344055:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949160.885021:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:1.0:1784949160.885031:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:2.0:1784949160.885042:0:344046:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:1.0:1784949160.885067:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000400:00000100:3.0:1784949160.885079:0:344057:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.3@tcp at host 111.222.173.3:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949160.885092:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000200:3.0:1784949160.885104:0:344057:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.3@tcp: down 00000400:00000001:1.0:1784949160.885106:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949160.885116:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949160.885121:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000200:3.0:1784949160.885122:0:344057:0:(router.c:1888:lnet_notify()) Old news 00000400:00000010:1.0:1784949160.885131:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000800:00000100:3.0:1784949160.885143:0:344057:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000400:00000200:1.0:1784949160.885145:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000010:3.0:1784949160.885159:0:344057:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000de6364cf. 00000400:00000200:1.0:1784949160.885162:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:1.0:1784949160.885179:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000be26f34f. 00000400:00000200:3.0:1784949160.885181:0:344057:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949160.885192:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:3.0:1784949160.885198:0:344057:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949160.885209:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:3.0:1784949160.885219:0:344057:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949160.885221:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000400:00000200:3.0:1784949160.885243:0:344057:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:1.0:1784949160.885244:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:1.0:1784949160.885260:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949160.885263:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949160.885266:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000200:3.0:1784949160.885268:0:344057:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:1.0:1784949160.885277:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000400:00000200:1.0:1784949160.885283:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949160.885298:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:1.0:1784949160.885302:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000008fae13bf. 00000400:00000200:1.0:1784949160.885310:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:1.0:1784949160.885314:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:1.0:1784949160.885317:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000400:00000200:1.0:1784949160.885321:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:1.0:1784949160.885324:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949160.885325:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949160.885327:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:1.0:1784949160.885329:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000400:00000200:1.0:1784949160.885331:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949160.885334:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:1.0:1784949160.885336:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006200396a. 00000400:00000200:1.0:1784949160.885342:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:1.0:1784949160.885345:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:1.0:1784949160.885348:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00010000:00000001:2.0:1784949161.204044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949161.204053:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949161.204062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949161.204066:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949161.204070:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949161.204073:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949161.204078:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949161.204080:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949161.204083:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949161.204086:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949161.204180:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949161.204187:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949161.204192:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:3.0:1784949161.588154:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949161.588169:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000af276f2a. 00000400:00000010:3.0:1784949161.588182:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf7eadde. 00000400:00000010:3.0:1784949161.588191:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bb55de7f. 00000400:00000010:3.0:1784949161.588203:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000061508776. 00000400:00000200:3.0:1784949161.588209:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949161.588232:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949161.588240:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949161.588246:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3703:3703:242:4294967295] 192.168.202.137@tcp LPNI seq info [2372:2372:8:4294967295] 00000400:00000200:3.0:1784949161.588256:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949161.588266:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949161.588274:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949161.588281:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000800:00000200:3.0:1784949161.588291:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949161.588304:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949161.588311:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949161.588339:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11435 health :500 00000800:00000200:2.0:1784949161.588580:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949161.588590:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:2.0:1784949161.588601:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949161.588642:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949161.588655:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949161.588662:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949161.588684:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949161.588698:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949161.588703:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949161.588710:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949161.588718:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949161.588735:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949161.588748:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949161.588763:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949161.588771:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3704:3704:242:4294967295] 192.168.202.137@tcp LPNI seq info [2373:2373:8:4294967295] 00000400:00000200:2.0:1784949161.588785:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949161.588797:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949161.588805:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000001ee120ca. 00000800:00000200:2.0:1784949161.588816:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949161.588829:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949161.588837:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000001ee120ca type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949161.588943:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949161.589040:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949161.589044:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949161.589051:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000001ee120ca. 00000400:00000200:2.0:1784949161.589056:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949161.589058:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949161.589067:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949161.589068:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949161.589079:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949161.589094:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000001:0.0:1784949161.589107:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949161.589118:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x90cd 00000800:00000001:1.0:1784949161.589128:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949161.589143:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949161.589146:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949161.589151:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949161.589159:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bb55de7f 00000400:00000010:1.0:1784949161.589163:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bb55de7f. 00000400:00000200:1.0:1784949161.589167:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949161.589172:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000af276f2a. 00000400:00000010:1.0:1784949161.589176:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf7eadde. 00000800:00000001:1.0:1784949161.589185:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949161.589188:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949162.228025:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949162.228033:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949162.228041:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949162.228043:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949162.228162:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949162.228169:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949162.228174:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949163.252016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949163.252023:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949163.252030:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949163.252031:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949163.252151:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949163.252160:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949163.252166:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949164.276033:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949164.276043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949164.276051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949164.276053:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949164.276158:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949164.276177:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949164.276184:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949164.468031:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949164.468048:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949164.468053:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949164.468062:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949164.468067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949164.468073:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.468077:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949164.468186:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949164.468196:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949164.468204:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fc20b3d9. 00000020:00000040:1.0:1784949164.468212:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949164.468219:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949164.468223:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949164.468230:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949164.468240:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005c010cf6. 00000100:00000001:1.0:1784949164.468249:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949164.468260:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949164.468276:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949164.468279:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949164.468285:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc20b3d9] to pc [ptlrpcd_00_03+3] req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949164.468333:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949164.468337:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949164.468342:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949164.468346:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949164.468351:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949164.468355:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008223e251. 00000020:00000040:1.0:1784949164.468359:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949164.468362:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949164.468365:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949164.468367:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949164.468372:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fee8262a. 00000100:00000001:1.0:1784949164.468376:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.468380:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949164.468385:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949164.468388:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949164.468394:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949164.468404:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1784949164.468406:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949164.468408:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949164.468424:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008223e251] to pc [ptlrpcd_00_00+0] req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949164.468440:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949164.468446:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949164.468456:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949164.468460:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949164.468464:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949164.468471:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fc20b3d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289678720:0@lo:400: 00000100:00000001:1.0:1784949164.468477:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.468479:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949164.468485:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949164.468491:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11441) 02000000:00000001:2.0:1784949164.468492:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949164.468504:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949164.468510:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949164.468512:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949164.468528:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000002fbb6924. 00000100:00000001:0.0:1784949164.468533:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949164.468535:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949164.468541:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949164.468548:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:2.0:1784949164.468549:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001712b5b2. 00000100:00000001:1.0:1784949164.468554:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949164.468563:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949164.468565:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949164.468568:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000100:00000200:2.0:1784949164.468586:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289678720, portal 25 00000100:00000001:2.0:1784949164.468619:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949164.468621:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637812736 : -115544071738880 : ffff96e9cca5ea00) 00000100:00000001:0.0:1784949164.468626:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949164.468631:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949164.468632:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949164.468639:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949164.468645:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949164.468655:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.468658:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949164.468665:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289678848:192.168.202.137@tcp:400: 00000100:00000040:2.0:1784949164.468667:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949164.468677:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:0.0:1784949164.468678:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000200:2.0:1784949164.468686:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289678720, offset 0 02000000:00000001:0.0:1784949164.468688:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949164.468694:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949164.468699:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784949164.468701:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:0.0:1784949164.468707:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949164.468713:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949164.468720:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ebd65351. 00000400:00000200:2.0:1784949164.468727:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:0.0:1784949164.468733:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000200:0.0:1784949164.468741:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289678848, portal 10 00000100:00000001:0.0:1784949164.468744:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949164.468747:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637810048 : -115544071741568 : ffff96e9cca5df80) 00000400:00000200:2.0:1784949164.468751:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c380 00000100:00000040:0.0:1784949164.468762:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949164.468772:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x86b9 [2] + 0 00000100:00000001:0.0:1784949164.468793:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:2.0:1784949164.468795:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008c614621. 00000100:00000040:0.0:1784949164.468802:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949164.468809:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000236111b7. 00000400:00000200:2.0:1784949164.468810:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000fe7b2a7d 00000100:00000200:0.0:1784949164.468824:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289678848, offset 0 00000400:00000200:2.0:1784949164.468828:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949164.468837:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949164.468844:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949164.468855:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949164.468863:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949164.468870:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949164.468874:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949164.468879:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949164.468883:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3705:3705:242:4294967295] 192.168.202.137@tcp LPNI seq info [2374:2374:8:4294967295] 00000400:00000200:0.0:1784949164.468902:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000400:00000200:2.0:1784949164.468970:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fe7b2a7d 00000800:00000200:0.0:1784949164.468971:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949164.468978:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000fe7b2a7d. 00000800:00000010:0.0:1784949164.468986:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002a753d9d. 00000100:00000001:2.0:1784949164.468989:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949164.468997:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949164.469002:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000043c318de x1871651289678720 msgsize 224 00000800:00000200:0.0:1784949164.469012:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:2.0:1784949164.469023:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949164.469028:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:0.0:1784949164.469038:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949164.469046:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002a753d9d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949164.469049:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949164.469063:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:2.0:1784949164.469066:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000100:00000001:2.0:1784949164.469071:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949164.469076:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949164.469082:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949164.469098:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2200 usecs (3 jiffies) 00000100:00000001:2.0:1784949164.469099:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949164.469108:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949164.469110:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306103812 00000100:00000010:0.0:1784949164.469124:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:2.0:1784949164.469126:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.469135:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949164.469142:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949164.469147:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.469157:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306103851 00000100:00000001:0.0:1784949164.469162:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306103812 00000100:00000001:2.0:1784949164.469168:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306103812 00000100:00000010:0.0:1784949164.469177:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:2.0:1784949164.469182:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306103851 00000100:00000001:0.0:1784949164.469190:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949164.469198:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784949164.469205:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306103812 00000100:00100000:3.0:1784949164.469208:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289678720 00000100:00000001:2.0:1784949164.469210:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306103812 00000100:00000010:0.0:1784949164.469217:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 02000000:00000001:3.0:1784949164.469222:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949164.469230:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306103851 00000100:00000001:3.0:1784949164.469234:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:1.0:1784949164.469237:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949164.469240:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949164.469240:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949164.469257:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002a753d9d. 02000000:00000001:3.0:1784949164.469265:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949164.469267:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306103812 00000100:00000001:2.0:1784949164.469267:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306103812 00000400:00000200:1.0:1784949164.469270:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:0.0:1784949164.469294:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00100000:3.0:1784949164.469294:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289678720 00000100:00000001:2.0:1784949164.469295:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306103851 00000400:00000200:1.0:1784949164.469308:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784949164.469314:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949164.469317:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000400:00000200:1.0:1784949164.469320:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000236111b7 00000020:00000040:3.0:1784949164.469322:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949164.469330:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949164.469332:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306103812 00000400:00000010:1.0:1784949164.469341:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000236111b7. 00000020:00000001:3.0:1784949164.469341:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949164.469343:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.469347:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1784949164.469362:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949164.469365:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.469365:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949164.469366:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949164.469384:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949164.469388:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949164.469389:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949164.469390:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949164.469416:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949164.469417:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949164.469418:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949164.469428:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949164.469430:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949164.469440:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949164.469442:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949164.469443:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949164.469444:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949164.469450:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949164.469458:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1784949164.469465:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000100:00000001:2.0:1784949164.469467:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949164.469468:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.469480:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949164.469486:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949164.469489:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949164.469491:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949164.469493:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000100:00000001:2.0:1784949164.469494:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949164.469517:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949164.469519:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949164.469525:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a17860fa. 00000020:00000010:3.0:1784949164.469537:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000889858a. 00000800:00000001:1.0:1784949164.469539:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784949164.469553:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000800:00000001:1.0:1784949164.469557:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949164.469563:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949164.469578:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949164.469584:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949164.469622:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:1.0:1784949164.469630:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949164.469632:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949164.469644:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949164.469656:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c400 00000100:00000001:3.0:1784949164.469660:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949164.469672:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 172848 00000800:00000001:1.0:1784949164.469682:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949164.469686:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949164.469690:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949164.469698:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949164.469704:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949164.469711:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949164.469713:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949164.469732:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:1.0:1784949164.469734:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784949164.469750:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000068fdded5. 00000100:00000001:0.0:1784949164.469753:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784949164.469763:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 791 00000100:00000001:0.0:1784949164.469766:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:1.0:1784949164.469767:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000068fdded5 x1871651289678848 msgsize 224 00000100:00000040:3.0:1784949164.469774:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949164.469776:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949164.469789:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742028368 : -115538967523248 : ffff96eafce21050) 00000100:00000001:0.0:1784949164.469791:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949164.469801:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:3.0:1784949164.469814:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000043c318de x1871651289678720/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:70/0 lens 224/0 e 0 to 0 dl 1784949175 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949164.469827:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949164.469830:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949164.469836:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e4a5d0ee. 00000800:00000001:1.0:1784949164.469839:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949164.469842:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:1.0:1784949164.469846:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949164.469848:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e4a5d0ee. 00000100:00100000:3.0:1784949164.469869:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000043c318de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289678720:12345-0@lo:400: 00000100:00000200:3.0:1784949164.469876:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289678720 00000020:00000001:3.0:1784949164.469884:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949164.469892:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949164.469897:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949164.469903:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949164.469948:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784949164.469950:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949164.469958:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289678848 00000020:00000001:3.0:1784949164.469959:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949164.469970:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949164.469972:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949164.469980:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:3.0:1784949164.469982:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949164.469994:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949164.469995:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949164.470008:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949164.470009:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949164.470025:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:2.0:1784949164.470027:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289678848 00000100:00000001:3.0:1784949164.470039:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:2.0:1784949164.470040:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949164.470052:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000001:3.0:1784949164.470054:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000001:2.0:1784949164.470067:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:3.0:1784949164.470071:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008618f1b2. 00000020:00000040:2.0:1784949164.470074:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:3.0:1784949164.470082:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949164.470083:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949164.470097:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949164.470098:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949164.470121:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.470123:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000040:3.0:1784949164.470129:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289678720 00000020:00000010:2.0:1784949164.470132:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00010000:00000001:3.0:1784949164.470135:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949164.470152:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000200:3.0:1784949164.470159:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000043c318de x1871651289678720/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:70/0 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949164.470165:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000020:00000010:2.0:1784949164.470185:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001df3b40e. 00010000:00000001:3.0:1784949164.470187:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949164.470195:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949164.470197:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000059fdf11b. 00000100:00001000:3.0:1784949164.470201:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=36 v=5 (1 1 1 1) 00000020:00000010:2.0:1784949164.470207:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784949164.470213:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949164.470222:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:2.0:1784949164.470225:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949164.470227:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:2.0:1784949164.470233:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:3.0:1784949164.470236:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784949164.470239:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784949164.470241:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.470244:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949164.470248:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949164.470255:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949164.470256:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949164.470268:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949164.470275:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949164.470275:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000100:00000200:3.0:1784949164.470302:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289678720, offset 224 00000100:00000001:2.0:1784949164.470314:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949164.470316:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949164.470335:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949164.470339:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784949164.470354:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:2.0:1784949164.470359:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 767 00000100:00000040:2.0:1784949164.470372:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949164.470379:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637803776 : -115544071747840 : ffff96e9cca5c700) 00000400:00000200:3.0:1784949164.470391:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c380 00000100:00000040:2.0:1784949164.470399:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000068fdded5 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:70/0 lens 224/0 e 0 to 0 dl 1784949175 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949164.470416:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x90d1 [1] + 224 00000100:00000001:2.0:1784949164.470448:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949164.470450:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:2.0:1784949164.470454:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:2.0:1784949164.470462:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949164.470468:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:2.0:1784949164.470472:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000001:3.0:1784949164.470480:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949164.470489:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000068fdded5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289678848:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949164.470495:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949164.470511:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289678848 00000020:00000001:2.0:1784949164.470521:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949164.470534:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949164.470541:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949164.470549:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949164.470552:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949164.470558:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949164.470575:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949164.470586:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949164.470622:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949164.470626:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949164.470629:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949164.470632:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949164.470633:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949164.470639:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949164.470645:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:3.0:1784949164.470650:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 02000000:00000010:2.0:1784949164.470652:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f9fa0ef9. 00000400:00000010:3.0:1784949164.470659:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 02000000:00000001:2.0:1784949164.470660:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949164.470676:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949164.470679:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:2.0:1784949164.470682:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949164.470684:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00010000:00000040:2.0:1784949164.470692:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289678848 00010000:00000001:2.0:1784949164.470699:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949164.470700:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949164.470710:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000200:2.0:1784949164.470714:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000068fdded5 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:70/0 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949164.470717:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949164.470729:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949164.470743:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949164.470753:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949164.470763:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1784949164.470764:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:3.0:1784949164.470766:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000043c318de x1871651289678720/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:70/0 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949164.470782:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000001:2.0:1784949164.470782:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949164.470788:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1784949164.470794:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000043c318de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289678720:12345-0@lo:400: Request processed in 957us (1784us total) trans 0 rc 0/0 00000400:00000010:1.0:1784949164.470802:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001712b5b2. 00000100:00001000:2.0:1784949164.470804:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=31 v=5 (1 1 1 1) 00000400:00000200:1.0:1784949164.470819:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000100:00100000:3.0:1784949164.470820:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 791 00000100:00000001:2.0:1784949164.470823:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1784949164.470841:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000100:00000040:3.0:1784949164.470843:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000040:2.0:1784949164.470848:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949164.470859:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949164.470861:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949164.470872:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784949164.470876:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:1.0:1784949164.470886:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949164.470888:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000010:3.0:1784949164.470891:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000889858a. 02000000:00000001:2.0:1784949164.470894:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949164.470948:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949164.470949:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:2.0:1784949164.470959:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:3.0:1784949164.470969:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a17860fa. 00000100:00000040:2.0:1784949164.470983:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949164.470988:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:3.0:1784949164.470988:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000010:2.0:1784949164.470992:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000100:00000001:1.0:1784949164.471002:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949164.471009:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:2.0:1784949164.471012:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289678848, offset 224 02000000:00000001:3.0:1784949164.471012:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:1.0:1784949164.471014:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000010:3.0:1784949164.471029:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008618f1b2. 00000100:00000001:1.0:1784949164.471030:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784949164.471044:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949164.471047:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949164.471050:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:1.0:1784949164.471064:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949164.471066:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949164.471068:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00001000:1.0:1784949164.471074:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=36 v=5 (5 5 5 5) 00000100:00000200:3.0:1784949164.471089:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:1.0:1784949164.471093:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=36 v=5 (1 1 1 1) 00000400:00000200:2.0:1784949164.471104:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3706:3706:242:4294967295] 192.168.202.137@tcp LPNI seq info [2375:2375:8:4294967295] 00000100:00000001:1.0:1784949164.471107:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:3.0:1784949164.471107:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e686bcb0. 00000100:00000001:1.0:1784949164.471117:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949164.471123:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a1933a2b. 00010000:00000001:1.0:1784949164.471124:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949164.471134:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949164.471143:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949164.471151:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:1.0:1784949164.471156:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949164.471172:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949164.471184:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1784949164.471184:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000001ee120ca. 00000100:00000001:1.0:1784949164.471196:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949164.471202:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949164.471203:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:1.0:1784949164.471211:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949164.471220:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949164.471230:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000001ee120ca type 1, nob 320 niov 1 nkiov 1 00000100:00100000:1.0:1784949164.471246:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fc20b3d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289678720:0@lo:400: 00000100:00000001:1.0:1784949164.471254:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949164.471256:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949164.471258:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fc20b3d9 x1871651289678720/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949164.471261:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949164.471274:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949164.471277:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:2.0:1784949164.471281:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 02000000:00000001:1.0:1784949164.471284:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1784949164.471286:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949164.471290:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000002fbb6924. 00000020:00000001:2.0:1784949164.471293:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 02000000:00000001:1.0:1784949164.471299:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:2.0:1784949164.471305:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000068fdded5 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:70/0 lens 224/224 e 0 to 0 dl 1784949175 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949164.471307:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949164.471313:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949164.471323:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949164.471328:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005c010cf6. 00000100:00100000:2.0:1784949164.471336:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000068fdded5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289678848:12345-192.168.202.137@tcp:400: Request processed in 882us (1562us total) trans 0 rc 0/0 02000000:00000001:1.0:1784949164.471338:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949164.471344:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949164.471350:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fc20b3d9. 00000100:00000001:1.0:1784949164.471356:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00100000:2.0:1784949164.471357:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 767 00000100:00000001:1.0:1784949164.471360:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949164.471366:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949164.471367:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949164.471377:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949164.471382:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949164.471383:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:1.0:1784949164.471390:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949164.471395:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000059fdf11b. 00000100:00000001:1.0:1784949164.471397:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949164.471404:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949164.471408:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949164.471415:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001df3b40e. 00000020:00000040:2.0:1784949164.471422:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949164.471428:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949164.471440:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949164.471446:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000001ee120ca. 00000400:00000200:0.0:1784949164.471454:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949164.471467:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949164.471476:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:0.0:1784949164.471482:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000800:00000001:3.0:1784949164.471483:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949164.471493:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949164.471497:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949164.471507:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949164.471514:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949164.471519:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949164.471532:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949164.471547:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c400 00000400:00000200:3.0:1784949164.471556:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x90d9 [1] + 224 00000800:00000001:3.0:1784949164.471563:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949164.471576:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949164.471579:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949164.471584:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949164.471617:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949164.471624:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949164.471637:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949164.471661:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949164.471669:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949164.471673:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949164.471713:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949164.471718:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949164.471724:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949164.471726:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949164.471730:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ebd65351. 00000400:00000200:0.0:1784949164.471736:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949164.471737:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784949164.471741:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949164.471748:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949164.471761:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949164.471771:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949164.471772:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949164.471775:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949164.471777:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949164.471780:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949164.471782:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949164.471785:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949164.471791:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=31 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949164.471796:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=31 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949164.471799:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949164.471801:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949164.471803:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949164.471807:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949164.471817:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949164.471820:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949164.471822:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949164.471824:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949164.471826:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949164.471829:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949164.471837:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949164.471841:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949164.471843:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949164.471846:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949164.471856:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289678848:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949164.471861:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949164.471863:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949164.471865:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008223e251 x1871651289678848/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949164.471873:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949164.471875:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949164.471877:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949164.471882:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949164.471884:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949164.471885:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949164.471888:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949164.471890:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fee8262a. 02000000:00000001:0.0:1784949164.471893:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949164.471895:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949164.471896:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008223e251. 00000100:00000001:0.0:1784949164.471900:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949164.471901:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949164.471937:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949164.471943:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949164.471944:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949164.471947:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949164.471949:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949164.512003:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306103851, at 4306103852 00000100:00000010:2.0:1784949164.512019:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:1.0:1784949164.512039:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306103851, at 4306103852 00000100:00000040:0.0:1784949164.512051:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306103851, at 4306103852 00000100:00000010:1.0:1784949164.512056:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000040:3.0:1784949164.512065:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306103851, at 4306103852 00000100:00000010:0.0:1784949164.512066:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:3.0:1784949164.512092:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00010000:00000001:2.0:1784949165.300050:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949165.300063:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949165.300075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949165.300078:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949165.300228:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949165.300239:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949165.300245:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949165.492023:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949165.492040:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949165.492043:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949165.492050:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949165.492055:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949165.492061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949165.492066:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949166.324007:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949166.324015:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949166.324022:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949166.324024:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949166.324135:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949166.324142:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949166.324147:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949166.516088:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949166.516110:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949166.516113:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949166.516121:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949166.516126:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949166.516131:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949166.516134:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949167.348056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949167.348066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949167.348074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949167.348076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949167.348212:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949167.348224:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949167.348230:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949167.540071:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949167.540095:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949167.540100:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949167.540110:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949167.540119:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949167.540128:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949167.540134:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949168.372047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949168.372057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949168.372066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949168.372068:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949168.372193:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949168.372200:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949168.372205:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949168.564164:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949168.564188:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949168.564191:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949168.564200:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949168.564204:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949168.564209:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949168.564213:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949169.396041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949169.396050:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949169.396057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949169.396059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949169.396147:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949169.396158:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949169.396164:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949169.588081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949169.588105:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949169.588108:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949169.588117:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949169.588122:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949169.588126:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949169.588128:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949169.588136:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949169.588142:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949169.588159:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949169.588171:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949169.588180:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949169.588184:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949169.588194:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949169.588206:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005a1c05a3. 00000100:00000001:0.0:1784949169.588217:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949169.588232:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949169.588250:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949169.588254:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949169.588261:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_01+1] req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949169.588311:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949169.588315:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949169.588320:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949169.588323:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949169.588327:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949169.588333:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949169.588336:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949169.588339:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949169.588341:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949169.588344:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949169.588350:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949169.588355:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949169.588359:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949169.588366:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1784949169.588368:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:3.0:1784949169.588377:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.588385:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949169.588389:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949169.588397:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949169.588402:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949169.588404:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949169.588405:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949169.588410:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949169.588418:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949169.588429:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_02+2] req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949169.588430:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949169.588446:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949169.588462:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949169.588465:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949169.588471:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949169.588476:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949169.588478:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949169.588481:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949169.588496:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289678976:0@lo:400: 00000100:00000040:0.0:1784949169.588498:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11446) 00000100:00000001:1.0:1784949169.588508:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949169.588511:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949169.588516:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949169.588522:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1784949169.588523:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949169.588530:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949169.588535:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784949169.588542:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784949169.588569:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000072ce5ef0. 02000000:00000001:3.0:1784949169.588582:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949169.588598:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949169.588603:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:3.0:1784949169.588604:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a23805e4. 00000100:00000001:2.0:1784949169.588609:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949169.588610:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949169.588620:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 02000000:00000001:1.0:1784949169.588623:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949169.588634:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784949169.588638:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289678976, portal 25 02000000:00000001:1.0:1784949169.588641:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949169.588647:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949169.588653:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000100:00100000:1.0:1784949169.588658:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289679104:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949169.588670:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1784949169.588670:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949169.588682:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949169.588692:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949169.588694:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949169.588704:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1784949169.588712:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949169.588717:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00000200:3.0:1784949169.588723:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289678976, offset 0 02000000:00000010:1.0:1784949169.588725:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 02000000:00000001:1.0:1784949169.588734:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949169.588758:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000502cbfd9. 00000400:00000200:3.0:1784949169.588758:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:1.0:1784949169.588781:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000400:00000200:3.0:1784949169.588794:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:1.0:1784949169.588796:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289679104, portal 10 00000100:00000001:1.0:1784949169.588807:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949169.588811:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000400:00000200:3.0:1784949169.588817:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c480 00000400:00000200:3.0:1784949169.588833:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x86d5 [2] + 0 00000100:00000040:1.0:1784949169.588841:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949169.588845:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000003793f8ab. 00000100:00000001:1.0:1784949169.588863:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949169.588864:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c55bc68f 00000100:00000040:1.0:1784949169.588872:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949169.588878:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000200:3.0:1784949169.588882:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1784949169.588883:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289679104, offset 0 00000400:00000200:1.0:1784949169.588942:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949169.588953:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949169.588966:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949169.588974:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949169.588984:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949169.588989:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949169.588993:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949169.588999:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3707:3707:242:4294967295] 192.168.202.137@tcp LPNI seq info [2376:2376:8:4294967295] 00000400:00000200:1.0:1784949169.589016:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949169.589026:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949169.589032:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c55bc68f 00000800:00000010:1.0:1784949169.589034:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005c010cf6. 00000400:00000010:3.0:1784949169.589051:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000c55bc68f. 00000800:00000200:1.0:1784949169.589058:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949169.589073:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784949169.589076:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:1.0:1784949169.589084:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005c010cf6 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:3.0:1784949169.589084:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949169.589100:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000cfaa2b5b x1871651289678976 msgsize 224 00000100:00100000:3.0:1784949169.589110:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784949169.589114:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949169.589123:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949169.589131:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2202 usecs (3 jiffies) 00000100:00000001:3.0:1784949169.589137:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949169.589138:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306108932 00000100:00000010:1.0:1784949169.589152:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000400:00000200:3.0:1784949169.589163:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000100:00000001:1.0:1784949169.589166:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000400:00000010:3.0:1784949169.589170:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00000001:3.0:1784949169.589182:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949169.589187:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306108932 00000100:00000200:3.0:1784949169.589196:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949169.589200:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:1.0:1784949169.589218:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949169.589223:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949169.589238:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289678976 00000100:00000001:3.0:1784949169.589243:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949169.589244:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306108932 02000000:00000001:0.0:1784949169.589246:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949169.589259:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000010:1.0:1784949169.589261:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000040:3.0:1784949169.589263:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949169.589270:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949169.589277:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 02000000:00000001:0.0:1784949169.589289:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949169.589291:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306108932 00000100:00100000:0.0:1784949169.589309:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289678976 00000100:00000010:1.0:1784949169.589318:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:3.0:1784949169.589320:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949169.589321:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949169.589337:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949169.589339:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949169.589340:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000001:0.0:1784949169.589351:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949169.589357:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1784949169.589359:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949169.589364:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949169.589381:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949169.589383:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949169.589386:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306108971 00000800:00000200:2.0:1784949169.589388:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1784949169.589415:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949169.589418:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949169.589418:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005c010cf6. 00000100:00000001:3.0:1784949169.589421:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306108932 00000100:00000001:1.0:1784949169.589440:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949169.589443:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949169.589445:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306108971 00000400:00000200:2.0:1784949169.589449:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:0.0:1784949169.589455:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949169.589457:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949169.589459:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306108932 00000100:00000001:1.0:1784949169.589475:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949169.589481:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949169.589481:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949169.589482:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306108971 00000100:00000001:1.0:1784949169.589485:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949169.589499:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949169.589506:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306108932 00000020:00000010:0.0:1784949169.589508:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949169.589511:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949169.589517:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000100:00000001:3.0:1784949169.589522:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306108971 00000100:00000001:1.0:1784949169.589527:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949169.589529:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000020:00000010:0.0:1784949169.589533:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000100:00000001:3.0:1784949169.589539:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306108932 00000100:00000001:2.0:1784949169.589577:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:0.0:1784949169.589580:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000006fd9d4e. 00000100:00000001:3.0:1784949169.589582:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949169.589596:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000200:2.0:1784949169.589598:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949169.589598:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949169.589618:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949169.589621:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949169.589631:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949169.589637:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949169.589641:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949169.589652:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000800:00000001:1.0:1784949169.589656:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949169.589663:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949169.589672:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949169.589674:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949169.589674:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949169.589695:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949169.589708:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949169.589717:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949169.589735:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949169.589740:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949169.589752:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949169.589754:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c500 00000400:00000200:1.0:1784949169.589786:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 173072 00000100:00000001:0.0:1784949169.589788:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949169.589794:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949169.589810:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 792 00000800:00000001:1.0:1784949169.589814:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949169.589817:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949169.589824:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742035536 : -115538967516080 : ffff96eafce22c50) 00000100:00000040:0.0:1784949169.589832:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000cfaa2b5b x1871651289678976/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:75/0 lens 224/0 e 0 to 0 dl 1784949180 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949169.589835:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949169.589850:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949169.589874:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949169.589890:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949169.589895:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:1.0:1784949169.589949:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fc20b3d9. 00000100:00000001:0.0:1784949169.589955:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949169.589956:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000fc20b3d9 x1871651289679104 msgsize 224 00000020:00000010:0.0:1784949169.589972:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00100000:1.0:1784949169.589972:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949169.589984:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949169.589990:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949169.589994:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00100000:0.0:1784949169.590000:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000cfaa2b5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289678976:12345-0@lo:400: 00000100:00000200:0.0:1784949169.590007:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289678976 00000800:00000001:1.0:1784949169.590009:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949169.590013:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:1.0:1784949169.590019:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949169.590023:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949169.590029:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949169.590034:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949169.590038:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949169.590044:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949169.590049:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949169.590054:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949169.590058:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949169.590062:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949169.590066:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949169.590066:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949169.590083:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289679104 00000100:00000001:0.0:1784949169.590084:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949169.590103:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:0.0:1784949169.590104:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949169.590117:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000010:0.0:1784949169.590121:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000b08bf97. 00000100:00000001:2.0:1784949169.590122:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949169.590134:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949169.590138:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949169.590140:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949169.590149:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949169.590149:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289679104 00010000:00000040:0.0:1784949169.590169:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289678976 00000020:00000001:2.0:1784949169.590170:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949169.590178:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000001:0.0:1784949169.590184:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:2.0:1784949169.590185:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949169.590193:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000200:0.0:1784949169.590194:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000cfaa2b5b x1871651289678976/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:75/0 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949169.590215:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:0.0:1784949169.590220:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949169.590228:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949169.590230:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00001000:0.0:1784949169.590236:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=41 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949169.590249:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949169.590251:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:0.0:1784949169.590255:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949169.590261:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000010:2.0:1784949169.590266:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 02000000:00000001:0.0:1784949169.590267:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949169.590275:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949169.590283:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949169.590283:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949169.590299:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:2.0:1784949169.590303:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000400:00000010:0.0:1784949169.590307:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000200:0.0:1784949169.590315:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289678976, offset 224 00000020:00000010:2.0:1784949169.590316:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001df3b40e. 00000020:00000010:2.0:1784949169.590328:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000059fdf11b. 00000400:00000200:0.0:1784949169.590329:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:2.0:1784949169.590347:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000200:0.0:1784949169.590358:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:2.0:1784949169.590362:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949169.590368:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949169.590372:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784949169.590377:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c480 00000100:00000001:2.0:1784949169.590382:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949169.590392:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x90ed [1] + 224 00000100:00000001:2.0:1784949169.590396:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:0.0:1784949169.590402:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949169.590402:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949169.590408:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949169.590422:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949169.590428:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949169.590433:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949169.590438:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949169.590461:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949169.590468:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949169.590480:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949169.590498:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 768 00000100:00000040:2.0:1784949169.590503:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949169.590506:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637812736 : -115544071738880 : ffff96e9cca5ea00) 00000100:00000001:0.0:1784949169.590513:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1784949169.590515:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000fc20b3d9 x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:75/0 lens 224/0 e 0 to 0 dl 1784949180 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949169.590530:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000100:00000001:2.0:1784949169.590535:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:0.0:1784949169.590536:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000020:00000010:2.0:1784949169.590543:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:0.0:1784949169.590571:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949169.590575:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949169.590580:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000010:2.0:1784949169.590585:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000001:0.0:1784949169.590599:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949169.590612:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000fc20b3d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289679104:12345-192.168.202.137@tcp:400: 00000100:00000040:0.0:1784949169.590622:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949169.590643:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784949169.590644:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:2.0:1784949169.590646:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289679104 00000100:00000001:1.0:1784949169.590651:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784949169.590654:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949169.590658:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949169.590661:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:2.0:1784949169.590666:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000001:1.0:1784949169.590667:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1784949169.590671:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949169.590673:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cfaa2b5b x1871651289678976/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:75/0 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949169.590677:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1784949169.590688:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:1.0:1784949169.590689:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a23805e4. 00000020:00000001:2.0:1784949169.590700:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:1.0:1784949169.590708:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000020:00000001:2.0:1784949169.590722:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000010:1.0:1784949169.590724:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000020:00000001:2.0:1784949169.590738:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.590742:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784949169.590745:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000cfaa2b5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289678976:12345-0@lo:400: Request processed in 783us (1634us total) trans 0 rc 0/0 00000020:00000001:2.0:1784949169.590755:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949169.590766:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:1.0:1784949169.590769:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949169.590781:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949169.590796:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 792 00000020:00000001:2.0:1784949169.590799:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:0.0:1784949169.590815:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949169.590816:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949169.590830:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:2.0:1784949169.590832:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000200:1.0:1784949169.590833:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949169.590835:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000010:2.0:1784949169.590846:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001e606a96. 00000020:00000010:0.0:1784949169.590848:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000006fd9d4e. 02000000:00000001:2.0:1784949169.590864:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949169.590867:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949169.590872:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949169.590874:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949169.590883:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000100:00000001:1.0:1784949169.590884:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949169.590885:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949169.590902:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289679104 00000100:00000001:1.0:1784949169.590947:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949169.590953:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949169.590957:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:0.0:1784949169.590960:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949169.590962:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949169.590970:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949169.590983:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:0.0:1784949169.590990:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:1.0:1784949169.590990:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=41 v=5 (5 5 5 5) 00010000:00000200:2.0:1784949169.590997:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000fc20b3d9 x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:75/0 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949169.591002:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000000b08bf97. 02000000:00000001:0.0:1784949169.591028:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:1.0:1784949169.591028:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=41 v=5 (1 1 1 1) 00010000:00000001:2.0:1784949169.591033:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949169.591039:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949169.591043:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949169.591049:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00001000:2.0:1784949169.591051:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=36 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949169.591053:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949169.591054:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949169.591079:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:2.0:1784949169.591081:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1784949169.591082:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ebd65351. 00010000:00000001:1.0:1784949169.591089:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949169.591093:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000400:00000010:0.0:1784949169.591097:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f29bca31. 00000100:00000001:1.0:1784949169.591098:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949169.591100:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949169.591113:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:1.0:1784949169.591118:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949169.591119:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949169.591129:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949169.591141:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949169.591151:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000100:00000001:1.0:1784949169.591156:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784949169.591162:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289679104, offset 224 00000100:00000001:1.0:1784949169.591163:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949169.591168:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949169.591178:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949169.591181:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784949169.591196:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289678976:0@lo:400: 00000100:00000001:1.0:1784949169.591206:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949169.591208:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949169.591211:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949169.591219:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289678976/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949169.591230:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3708:3708:242:4294967295] 192.168.202.137@tcp LPNI seq info [2377:2377:8:4294967295] 00000100:00000001:1.0:1784949169.591238:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949169.591243:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949169.591249:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000072ce5ef0. 02000000:00000001:1.0:1784949169.591255:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949169.591261:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949169.591266:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949169.591273:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949169.591279:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005a1c05a3. 00000400:00000200:2.0:1784949169.591279:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:1.0:1784949169.591289:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949169.591293:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949169.591297:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000800:00000200:2.0:1784949169.591299:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949169.591309:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000cf155122. 00000100:00000001:1.0:1784949169.591312:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949169.591316:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949169.591321:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949169.591323:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.591337:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:2.0:1784949169.591339:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949169.591342:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949169.591348:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000cf155122 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949169.591351:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949169.591359:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949169.591375:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949169.591383:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949169.591387:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949169.591390:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949169.591393:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949169.591398:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fc20b3d9 x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:75/0 lens 224/224 e 0 to 0 dl 1784949180 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949169.591410:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000fc20b3d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289679104:12345-192.168.202.137@tcp:400: Request processed in 876us (1453us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949169.591421:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 768 00000100:00000040:2.0:1784949169.591425:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949169.591428:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949169.591430:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949169.591436:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000059fdf11b. 00000020:00000010:2.0:1784949169.591441:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949169.591445:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001df3b40e. 00000020:00000040:2.0:1784949169.591450:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949169.591456:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949169.591718:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949169.591727:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000cf155122. 00000400:00000200:3.0:1784949169.591734:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949169.591745:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949169.591747:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949169.591757:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000800:00000001:2.0:1784949169.591763:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949169.591763:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000800:00000001:2.0:1784949169.591783:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949169.591784:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949169.591789:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949169.591790:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949169.591810:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949169.591824:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c500 00000400:00000200:2.0:1784949169.591832:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x90f5 [1] + 224 00000800:00000001:2.0:1784949169.591839:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949169.591855:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949169.591858:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949169.591863:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949169.591868:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949169.591872:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949169.591885:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949169.591932:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949169.591941:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949169.591946:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.591969:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949169.591974:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949169.591980:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949169.591982:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949169.591984:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000502cbfd9. 00000400:00000200:1.0:1784949169.591990:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949169.591992:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784949169.591996:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949169.592002:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949169.592013:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949169.592022:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949169.592023:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.592026:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949169.592028:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949169.592029:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949169.592031:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949169.592033:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949169.592040:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=36 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949169.592043:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=36 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949169.592046:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949169.592048:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949169.592050:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949169.592054:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949169.592063:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949169.592066:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949169.592068:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949169.592070:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949169.592071:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949169.592075:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949169.592083:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949169.592085:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949169.592087:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949169.592090:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949169.592098:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289679104:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949169.592104:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.592106:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949169.592109:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289679104/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949169.592117:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949169.592118:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949169.592120:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:1.0:1784949169.592124:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949169.592125:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949169.592126:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949169.592129:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949169.592130:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:1.0:1784949169.592133:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949169.592135:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949169.592137:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000001:1.0:1784949169.592142:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949169.592143:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.592145:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949169.592149:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949169.592151:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949169.592154:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949169.592156:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949169.632038:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306108971, at 4306108972 00000100:00000040:2.0:1784949169.632045:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306108971, at 4306108972 00000100:00000040:0.0:1784949169.632051:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306108971, at 4306108972 00000100:00000010:1.0:1784949169.632054:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:2.0:1784949169.632061:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:3.0:1784949169.632080:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306108971, at 4306108972 00000100:00000010:0.0:1784949169.632086:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:3.0:1784949169.632105:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00010000:00000001:2.0:1784949170.420061:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949170.420071:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949170.420079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949170.420081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949170.420203:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949170.420210:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949170.420214:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949170.612055:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949170.612078:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949170.612081:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949170.612088:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949170.612093:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949170.612098:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949170.612102:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949171.444057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949171.444066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949171.444075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949171.444079:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949171.444083:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949171.444087:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949171.444090:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949171.444093:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949171.444096:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949171.444098:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949171.444226:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949171.444237:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949171.444243:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949171.636109:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949171.636126:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949171.636130:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949171.636136:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949171.636142:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949171.636148:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949171.636152:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949172.468065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949172.468076:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949172.468088:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949172.468091:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949172.468220:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949172.468227:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949172.468232:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949172.660186:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949172.660201:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949172.660203:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949172.660210:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949172.660214:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949172.660218:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949172.660221:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949173.492035:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949173.492044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949173.492052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949173.492054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949173.492192:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949173.492203:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949173.492209:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949173.684032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949173.684046:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949173.684049:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949173.684056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949173.684059:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949173.684065:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949173.684070:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949174.516048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949174.516057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949174.516065:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949174.516067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949174.516192:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949174.516199:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949174.516203:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949174.708078:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949174.708093:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949174.708095:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.708103:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949174.708108:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949174.708112:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949174.708116:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949174.708168:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949174.708184:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949174.708197:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046cb422a. 00000020:00000040:3.0:1784949174.708208:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949174.708217:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949174.708223:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949174.708232:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949174.708247:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000001:3.0:1784949174.708259:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949174.708275:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949174.708293:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949174.708297:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949174.708307:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046cb422a] to pc [ptlrpcd_00_03+3] req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949174.708357:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949174.708365:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949174.708371:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:1.0:1784949174.708372:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949174.708379:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1784949174.708383:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:3.0:1784949174.708389:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949174.708391:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1784949174.708399:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000100:00000001:0.0:1784949174.708401:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949174.708403:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1784949174.708411:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949174.708414:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.708415:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949174.708426:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949174.708435:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949174.708437:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949174.708452:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949174.708455:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949174.708459:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:3.0:1784949174.708463:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949174.708470:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949174.708476:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949174.708486:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949174.708544:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949174.708554:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949174.708572:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949174.708575:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949174.708580:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949174.708580:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949174.708594:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949174.708602:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006d868603] to pc [ptlrpcd_00_00+0] req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949174.708609:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289679232:0@lo:400: 00000100:00000001:1.0:1784949174.708636:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949174.708651:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949174.708656:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949174.708660:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949174.708667:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004f865a97. 02000000:00000001:1.0:1784949174.708672:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949174.708681:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000502cbfd9. 00000400:00000010:1.0:1784949174.708692:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000200:1.0:1784949174.708702:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289679232, portal 25 00000100:00000001:0.0:1784949174.708706:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949174.708707:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949174.708713:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554953472 : -115544154598144 : ffff96e9c7b59500) 00000100:00000001:0.0:1784949174.708715:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949174.708722:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949174.708728:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949174.708735:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949174.708738:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949174.708747:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949174.708761:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949174.708771:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784949174.708772:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949174.708778:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949174.708780:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 02000000:00000001:0.0:1784949174.708784:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000200:1.0:1784949174.708786:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289679232, offset 0 02000000:00000001:0.0:1784949174.708790:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949174.708796:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949174.708801:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:0.0:1784949174.708807:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289679360:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949174.708816:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949174.708821:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949174.708826:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949174.708830:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:0.0:1784949174.708833:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949174.708845:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949174.708849:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949174.708857:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c580 00000400:00000010:0.0:1784949174.708862:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bb684072. 00000400:00000200:1.0:1784949174.708871:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x86f1 [2] + 0 00000400:00000010:1.0:1784949174.708878:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a51bb92. 00000400:00000200:1.0:1784949174.708887:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000008c74c22d 00000400:00000010:0.0:1784949174.708896:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000400:00000200:1.0:1784949174.708897:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1784949174.708944:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289679360, portal 10 00000100:00000001:0.0:1784949174.708949:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949174.708952:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554951680 : -115544154599936 : ffff96e9c7b58e00) 00000100:00000040:0.0:1784949174.708967:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949174.708975:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949174.708983:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949174.708990:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008c74c22d 00000400:00000010:1.0:1784949174.709002:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000008c74c22d. 00000100:00000040:3.0:1784949174.709005:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11451) 00000100:00000001:0.0:1784949174.709007:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949174.709022:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:0.0:1784949174.709025:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:1.0:1784949174.709030:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:0.0:1784949174.709033:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000100:00000040:1.0:1784949174.709038:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000774f7e3e x1871651289679232 msgsize 224 00000100:00000200:0.0:1784949174.709040:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289679360, offset 0 00000100:00100000:1.0:1784949174.709052:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949174.709059:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784949174.709060:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949174.709083:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949174.709083:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949174.709095:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949174.709105:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949174.709113:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949174.709124:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949174.709131:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:0.0:1784949174.709133:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3709:3709:242:4294967295] 192.168.202.137@tcp LPNI seq info [2378:2378:8:4294967295] 00000100:00100000:3.0:1784949174.709140:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289679232 02000000:00000001:3.0:1784949174.709148:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949174.709156:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949174.709163:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949174.709175:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:3.0:1784949174.709178:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949174.709191:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289679232 00000800:00000200:0.0:1784949174.709193:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949174.709195:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000800:00000010:0.0:1784949174.709216:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000400:00000010:1.0:1784949174.709217:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000020:00000001:3.0:1784949174.709217:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949174.709230:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784949174.709239:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949174.709240:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:0.0:1784949174.709247:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000040:3.0:1784949174.709258:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000200:1.0:1784949174.709262:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949174.709267:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:0.0:1784949174.709272:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:3.0:1784949174.709281:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:0.0:1784949174.709291:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949174.709301:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949174.709315:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:1.0:1784949174.709319:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949174.709322:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949174.709326:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949174.709330:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949174.709339:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000100:00000040:0.0:1784949174.709345:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2205 usecs (3 jiffies) 00000100:00000001:0.0:1784949174.709360:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306114052 00000020:00000010:3.0:1784949174.709365:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000052e1e01a. 00000100:00000001:1.0:1784949174.709368:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949174.709372:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:1.0:1784949174.709379:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784949174.709384:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000889858a. 00000100:00000001:0.0:1784949174.709388:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949174.709391:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949174.709395:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949174.709400:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306114091 00000100:00000001:0.0:1784949174.709403:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306114052 00000100:00000010:0.0:1784949174.709412:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:1.0:1784949174.709414:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306114052 00000100:00000040:3.0:1784949174.709419:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949174.709421:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949174.709430:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306114091 00000100:00000001:3.0:1784949174.709435:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949174.709440:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306114052 00000100:00000001:3.0:1784949174.709441:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949174.709444:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306114052 00000100:00000001:3.0:1784949174.709455:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949174.709459:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:0.0:1784949174.709470:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949174.709470:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306114091 00000100:00000001:3.0:1784949174.709476:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949174.709485:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306114052 00000100:00000010:0.0:1784949174.709494:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:1.0:1784949174.709495:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306114052 00000100:00000001:3.0:1784949174.709497:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.709524:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000800:00000200:2.0:1784949174.709533:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949174.709534:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949174.709538:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306114091 00000100:00000001:0.0:1784949174.709548:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949174.709549:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000100:00000001:0.0:1784949174.709558:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949174.709559:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949174.709560:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306114052 00000400:00000200:2.0:1784949174.709564:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949174.709579:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949174.709579:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949174.709582:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949174.709600:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949174.709600:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949174.709603:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1784949174.709615:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 793 00000100:00000001:0.0:1784949174.709617:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949174.709620:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949174.709629:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000100:00000040:3.0:1784949174.709637:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949174.709639:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949174.709639:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949174.709640:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000001:0.0:1784949174.709652:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949174.709653:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742033488 : -115538967518128 : ffff96eafce22450) 00000100:00000001:2.0:1784949174.709663:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949174.709670:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949174.709676:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000774f7e3e x1871651289679232/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:80/0 lens 224/0 e 0 to 0 dl 1784949185 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.709682:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949174.709686:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949174.709697:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949174.709718:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949174.709739:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949174.709751:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949174.709759:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949174.709769:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000400:00000200:1.0:1784949174.709781:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949174.709785:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949174.709790:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1784949174.709798:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000400:00000200:1.0:1784949174.709815:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c600 00000100:00000040:2.0:1784949174.709815:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949174.709830:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000774f7e3e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289679232:12345-0@lo:400: 00000400:00000200:1.0:1784949174.709860:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 173296 00000100:00000200:3.0:1784949174.709862:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289679232 00000020:00000001:3.0:1784949174.709876:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949174.709892:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000001:1.0:1784949174.709893:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949174.709963:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949174.709969:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949174.709971:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:1.0:1784949174.709975:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949174.709979:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784949174.709982:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784949174.709984:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1784949174.709992:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949174.709996:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949174.710004:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784949174.710006:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949174.710013:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949174.710015:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:3.0:1784949174.710017:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949174.710020:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:3.0:1784949174.710026:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000010:1.0:1784949174.710032:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bb35a642. 00000100:00000001:3.0:1784949174.710033:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949174.710044:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:1.0:1784949174.710046:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000bb35a642 x1871651289679360 msgsize 224 02000000:00000010:3.0:1784949174.710053:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003345e898. 02000000:00000001:3.0:1784949174.710066:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949174.710078:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784949174.710079:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949174.710095:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949174.710102:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289679232 00010000:00000001:3.0:1784949174.710107:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949174.710114:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000200:3.0:1784949174.710115:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000774f7e3e x1871651289679232/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:80/0 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949174.710126:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:3.0:1784949174.710132:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000800:00000001:1.0:1784949174.710135:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949174.710138:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949174.710146:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=46 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949174.710156:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949174.710164:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949174.710168:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:0.0:1784949174.710174:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:3.0:1784949174.710174:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949174.710184:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949174.710186:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289679360 00000100:00000001:3.0:1784949174.710192:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949174.710193:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949174.710208:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949174.710216:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949174.710218:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784949174.710236:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949174.710237:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00100000:0.0:1784949174.710260:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289679360 00000100:00000200:3.0:1784949174.710261:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289679232, offset 224 00000020:00000001:0.0:1784949174.710270:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949174.710275:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:3.0:1784949174.710283:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:0.0:1784949174.710285:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949174.710293:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949174.710300:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949174.710306:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784949174.710316:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949174.710321:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949174.710328:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:0.0:1784949174.710334:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949174.710335:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c580 00000020:00000010:0.0:1784949174.710347:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000400:00000200:3.0:1784949174.710359:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9109 [1] + 224 00000020:00000010:0.0:1784949174.710360:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000020:00000010:0.0:1784949174.710376:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000400:00000200:3.0:1784949174.710383:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:0.0:1784949174.710385:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:3.0:1784949174.710395:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:0.0:1784949174.710405:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949174.710406:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949174.710418:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000200:3.0:1784949174.710421:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.710423:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949174.710430:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.710449:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000040:3.0:1784949174.710453:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.710456:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949174.710463:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949174.710491:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949174.710496:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949174.710535:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784949174.710535:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000100:00000001:0.0:1784949174.710542:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:3.0:1784949174.710543:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00000001:3.0:1784949174.710556:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949174.710567:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00100000:0.0:1784949174.710569:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 769 00000100:00000001:2.0:1784949174.710581:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949174.710587:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949174.710589:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949174.710590:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949174.710595:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637806464 : -115544071745152 : ffff96e9cca5d180) 00000100:00000001:2.0:1784949174.710607:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949174.710610:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000040:0.0:1784949174.710615:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bb35a642 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:80/0 lens 224/0 e 0 to 0 dl 1784949185 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949174.710619:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949174.710627:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949174.710631:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949174.710632:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949174.710646:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949174.710654:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1784949174.710664:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:1.0:1784949174.710666:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949174.710670:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:1.0:1784949174.710686:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000502cbfd9. 00000020:00000010:0.0:1784949174.710687:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000040:3.0:1784949174.710692:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774f7e3e x1871651289679232/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:80/0 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.710704:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949174.710707:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949174.710719:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000020:00000010:0.0:1784949174.710723:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949174.710733:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949174.710752:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949174.710753:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bb35a642 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289679360:12345-192.168.202.137@tcp:400: 00000100:00100000:3.0:1784949174.710775:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000774f7e3e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289679232:12345-0@lo:400: Request processed in 1003us (1722us total) trans 0 rc 0/0 00000100:00000200:0.0:1784949174.710784:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289679360 00000020:00000001:0.0:1784949174.710799:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949174.710810:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949174.710826:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949174.710836:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949174.710839:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 793 00000020:00000001:0.0:1784949174.710840:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949174.710850:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000040:3.0:1784949174.710857:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:0.0:1784949174.710863:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949174.710873:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:0.0:1784949174.710875:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949174.710884:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:0.0:1784949174.710887:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949174.710899:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949174.710900:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784949174.710902:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000889858a. 00000020:00000001:0.0:1784949174.710960:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949174.710962:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949174.710964:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949174.710984:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949174.710986:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:3.0:1784949174.710988:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000052e1e01a. 00000100:00000001:0.0:1784949174.710993:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949174.710998:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:0.0:1784949174.711004:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949174.711005:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:3.0:1784949174.711007:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949174.711012:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949174.711014:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002816ea8d. 02000000:00000001:3.0:1784949174.711021:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:0.0:1784949174.711023:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949174.711024:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949174.711034:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000003345e898. 00000100:00000001:0.0:1784949174.711040:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949174.711042:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=46 v=5 (5 5 5 5) 02000000:00000001:3.0:1784949174.711045:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:0.0:1784949174.711049:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949174.711053:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784949174.711063:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=46 v=5 (1 1 1 1) 00010000:00000040:0.0:1784949174.711065:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289679360 00000100:00000200:3.0:1784949174.711068:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949174.711084:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00010000:00000001:0.0:1784949174.711085:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000010:3.0:1784949174.711087:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000382215bc. 00000100:00000001:1.0:1784949174.711092:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949174.711103:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000049294c87. 00010000:00000200:0.0:1784949174.711105:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bb35a642 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:80/0 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949174.711105:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949174.711117:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949174.711127:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949174.711140:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:1.0:1784949174.711141:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949174.711146:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949174.711157:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=41 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949174.711165:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949174.711170:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949174.711172:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949174.711178:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949174.711182:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000040:1.0:1784949174.711188:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.711190:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949174.711201:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949174.711207:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949174.711219:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1784949174.711220:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289679232:0@lo:400: 00000100:00000040:0.0:1784949174.711233:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949174.711236:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949174.711239:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000001:1.0:1784949174.711241:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949174.711250:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000046cb422a x1871651289679232/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949174.711251:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289679360, offset 224 00000400:00000200:0.0:1784949174.711271:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949174.711276:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949174.711281:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949174.711284:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004f865a97. 00000400:00000200:0.0:1784949174.711291:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:1.0:1784949174.711293:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949174.711300:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949174.711304:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:0.0:1784949174.711310:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3710:3710:242:4294967295] 192.168.202.137@tcp LPNI seq info [2379:2379:8:4294967295] 00000020:00000001:1.0:1784949174.711313:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949174.711320:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:1.0:1784949174.711328:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949174.711334:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:0.0:1784949174.711336:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000010:1.0:1784949174.711338:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000046cb422a. 00000100:00000001:1.0:1784949174.711350:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949174.711356:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949174.711357:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949174.711365:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784949174.711368:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009ff40a64. 00000100:00000001:1.0:1784949174.711374:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:0.0:1784949174.711382:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949174.711382:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949174.711391:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:0.0:1784949174.711398:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949174.711399:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949174.711407:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009ff40a64 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949174.711428:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949174.711433:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949174.711436:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949174.711438:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949174.711441:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949174.711445:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bb35a642 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:80/0 lens 224/224 e 0 to 0 dl 1784949185 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949174.711458:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bb35a642 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289679360:12345-192.168.202.137@tcp:400: Request processed in 787us (1409us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949174.711468:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 769 00000100:00000040:0.0:1784949174.711473:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949174.711476:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949174.711478:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949174.711485:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000020:00000010:0.0:1784949174.711491:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949174.711496:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000020:00000040:0.0:1784949174.711527:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949174.711531:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949174.711599:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949174.711606:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009ff40a64. 00000400:00000200:2.0:1784949174.711612:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949174.711619:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949174.711624:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949174.711626:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000400:00000010:2.0:1784949174.711631:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000100:00000001:2.0:1784949174.711636:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784949174.711639:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949174.711641:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949174.711648:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949174.711655:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949174.711665:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949174.711679:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c600 00000400:00000200:1.0:1784949174.711687:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9111 [1] + 224 00000800:00000001:1.0:1784949174.711694:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949174.711702:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949174.711705:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949174.711710:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949174.711715:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949174.711720:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949174.711732:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949174.711757:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949174.711764:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949174.711767:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.711806:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949174.711811:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949174.711816:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949174.711818:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949174.711821:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bb684072. 00000400:00000200:0.0:1784949174.711827:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949174.711830:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784949174.711833:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949174.711839:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949174.711851:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.711860:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949174.711862:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.711864:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949174.711867:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949174.711869:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949174.711870:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949174.711873:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949174.711879:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=41 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949174.711882:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=41 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949174.711886:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949174.711888:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949174.711890:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949174.711895:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949174.711951:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949174.711955:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949174.711958:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949174.711961:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949174.711963:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949174.711968:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.711978:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949174.711980:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949174.711982:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949174.711987:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949174.712005:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289679360:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949174.712010:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.712013:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949174.712015:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006d868603 x1871651289679360/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949174.712023:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949174.712026:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949174.712028:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949174.712033:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949174.712035:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949174.712037:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949174.712040:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949174.712042:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:0.0:1784949174.712045:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949174.712047:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949174.712049:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006d868603. 00000100:00000001:0.0:1784949174.712065:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949174.712066:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.712068:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949174.712072:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949174.712074:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949174.712077:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949174.712079:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949174.752048:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306114091, at 4306114092 00000100:00000010:0.0:1784949174.752063:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:2.0:1784949174.752079:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306114091, at 4306114092 00000100:00000010:2.0:1784949174.752096:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:1.0:1784949174.752102:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306114091, at 4306114092 00000100:00000040:3.0:1784949174.752105:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306114091, at 4306114092 00000100:00000010:1.0:1784949174.752124:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000010:3.0:1784949174.752132:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00010000:00000001:2.0:1784949175.540053:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949175.540064:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949175.540072:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949175.540074:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949175.540171:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949175.540180:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949175.540185:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949175.732072:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949175.732090:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949175.732093:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949175.732101:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949175.732108:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949175.732113:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949175.732118:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949176.564078:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949176.564090:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949176.564102:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949176.564105:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949176.564202:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949176.564208:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949176.564213:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949176.756032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949176.756051:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949176.756055:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949176.756062:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949176.756065:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949176.756070:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949176.756073:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949177.588029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949177.588041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949177.588049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949177.588050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949177.588148:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949177.588155:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949177.588159:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949177.780215:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949177.780234:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949177.780237:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949177.780245:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949177.780249:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949177.780254:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949177.780257:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949178.611983:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949178.611992:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949178.611998:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949178.611999:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949178.612053:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949178.612058:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949178.612061:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949178.803968:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949178.803983:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949178.803985:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949178.803990:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949178.803992:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949178.803996:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949178.803998:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949179.636017:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949179.636030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949179.636040:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949179.636041:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949179.636142:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949179.636168:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949179.636172:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949179.828011:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949179.828028:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949179.828032:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949179.828039:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949179.828043:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949179.828049:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.828052:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949179.828088:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949179.828098:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949179.828112:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949179.828123:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949179.828130:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949179.828134:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949179.828144:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949179.828157:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003a8a98bc. 00000100:00000001:0.0:1784949179.828167:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949179.828181:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949179.828198:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949179.828201:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949179.828209:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_01+1] req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949179.828242:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949179.828245:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949179.828250:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949179.828255:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949179.828277:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949179.828282:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949179.828286:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949179.828289:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949179.828291:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949179.828292:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949179.828300:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.828301:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949179.828309:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:0.0:1784949179.828310:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000057c7ab4. 00000100:00000001:0.0:1784949179.828319:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949179.828322:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949179.828336:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949179.828338:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949179.828349:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.828361:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949179.828368:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949179.828370:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949179.828376:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949179.828379:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949179.828385:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_02+2] req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949179.828386:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949179.828397:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949179.828409:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289679488:0@lo:400: 00000100:00000001:2.0:1784949179.828420:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949179.828429:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949179.828433:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949179.828437:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949179.828440:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949179.828446:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001df3b40e. 00000100:00000040:0.0:1784949179.828451:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11456) 02000000:00000001:2.0:1784949179.828452:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949179.828505:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949179.828510:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1784949179.828512:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008c614621. 00000100:00000001:1.0:1784949179.828514:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:2.0:1784949179.828526:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00000040:1.0:1784949179.828528:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949179.828539:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784949179.828541:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289679488, portal 25 00000100:00000001:3.0:1784949179.828548:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949179.828552:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949179.828553:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949179.828559:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.828559:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 02000000:00000001:1.0:1784949179.828569:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1784949179.828575:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949179.828576:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949179.828587:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949179.828599:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289679616:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949179.828612:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949179.828612:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949179.828618:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949179.828623:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949179.828624:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784949179.828629:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:2.0:1784949179.828633:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 02000000:00000010:1.0:1784949179.828638:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 00000100:00000200:2.0:1784949179.828643:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289679488, offset 0 02000000:00000001:1.0:1784949179.828644:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949179.828654:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000502cbfd9. 00000400:00000200:2.0:1784949179.828665:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:1.0:1784949179.828682:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000200:1.0:1784949179.828690:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289679616, portal 10 00000100:00000001:1.0:1784949179.828697:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:2.0:1784949179.828702:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949179.828703:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000100:00000040:1.0:1784949179.828718:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949179.828729:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784949179.828736:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c680 00000100:00000040:1.0:1784949179.828740:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949179.828750:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000400:00000200:2.0:1784949179.828759:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8715 [2] + 0 00000400:00000010:2.0:1784949179.828768:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d97ec0aa. 00000400:00000200:2.0:1784949179.828782:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000000b56e9af 00000100:00000200:1.0:1784949179.828789:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289679616, offset 0 00000400:00000200:2.0:1784949179.828799:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949179.828808:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949179.828815:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949179.828835:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949179.828848:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949179.828856:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949179.828863:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949179.828869:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949179.828876:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3711:3711:242:4294967295] 192.168.202.137@tcp LPNI seq info [2380:2380:8:4294967295] 00000400:00000200:1.0:1784949179.828898:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949179.828939:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949179.828947:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004b264730. 00000400:00000200:2.0:1784949179.828952:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000b56e9af 00000400:00000010:2.0:1784949179.828959:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000000b56e9af. 00000800:00000200:1.0:1784949179.828964:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949179.828971:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949179.828977:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:1.0:1784949179.828985:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:2.0:1784949179.828987:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000b4879340 x1871651289679488 msgsize 224 00000800:00000200:1.0:1784949179.828995:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004b264730 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:2.0:1784949179.829005:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949179.829010:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949179.829025:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949179.829030:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949179.829037:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000100:00000040:1.0:1784949179.829039:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2207 usecs (3 jiffies) 00000400:00000010:2.0:1784949179.829044:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:1.0:1784949179.829050:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306119172 00000100:00000001:2.0:1784949179.829052:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949179.829062:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949179.829063:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784949179.829079:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949179.829080:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949179.829083:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1784949179.829087:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289679488 00000100:00000040:2.0:1784949179.829093:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949179.829094:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949179.829097:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306119172 00000100:00000001:0.0:1784949179.829101:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949179.829108:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949179.829111:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000001:2.0:1784949179.829120:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949179.829122:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949179.829123:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949179.829127:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1784949179.829135:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289679488 00000100:00000001:1.0:1784949179.829138:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306119172 00000020:00000001:0.0:1784949179.829143:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949179.829144:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949179.829146:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000040:0.0:1784949179.829149:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784949179.829154:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306119211 00000020:00000001:0.0:1784949179.829157:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949179.829158:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000040:0.0:1784949179.829167:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949179.829171:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306119172 00000100:00000001:2.0:1784949179.829171:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306119172 00000020:00000001:0.0:1784949179.829174:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784949179.829184:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784949179.829184:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:2.0:1784949179.829185:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306119211 00000100:00000001:0.0:1784949179.829196:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949179.829199:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949179.829200:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306119172 00000020:00000010:0.0:1784949179.829206:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949179.829214:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949179.829215:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949179.829216:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306119211 00000100:00000001:1.0:1784949179.829225:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949179.829229:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949179.829232:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.829233:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306119172 00000100:00000001:1.0:1784949179.829240:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949179.829241:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000100:00000001:2.0:1784949179.829247:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306119211 00000020:00000010:0.0:1784949179.829254:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000001:1.0:1784949179.829256:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949179.829258:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306119172 00000800:00000200:3.0:1784949179.829261:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949179.829263:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:0.0:1784949179.829265:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949179.829267:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:3.0:1784949179.829271:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004b264730. 00000100:00000001:1.0:1784949179.829272:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949179.829276:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949179.829278:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949179.829284:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949179.829287:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949179.829288:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949179.829290:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949179.829297:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949179.829300:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949179.829302:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:3.0:1784949179.829302:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949179.829309:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949179.829311:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949179.829315:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:3.0:1784949179.829316:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000100:00000001:0.0:1784949179.829317:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.829326:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949179.829326:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000100:00000001:2.0:1784949179.829332:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949179.829336:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949179.829337:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949179.829338:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949179.829350:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949179.829351:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949179.829353:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949179.829363:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949179.829374:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949179.829375:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949179.829383:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949179.829386:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949179.829390:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:1.0:1784949179.829391:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949179.829397:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949179.829408:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949179.829416:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949179.829416:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949179.829418:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 794 00000400:00000200:1.0:1784949179.829419:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949179.829426:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1784949179.829432:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949179.829433:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949179.829442:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742030416 : -115538967521200 : ffff96eafce21850) 00000100:00000001:2.0:1784949179.829444:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949179.829453:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c700 00000100:00000040:0.0:1784949179.829485:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b4879340 x1871651289679488/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:85/0 lens 224/0 e 0 to 0 dl 1784949190 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949179.829498:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 173520 00000800:00000001:1.0:1784949179.829516:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949179.829518:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949179.829525:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000800:00000001:1.0:1784949179.829529:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949179.829530:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949179.829540:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000400:00000200:1.0:1784949179.829541:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949179.829554:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:0.0:1784949179.829560:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b4879340 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289679488:12345-0@lo:400: 00000100:00000001:1.0:1784949179.829565:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949179.829567:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289679488 00000100:00000200:1.0:1784949179.829570:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:0.0:1784949179.829574:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000010:1.0:1784949179.829579:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000473c8c31. 00000020:00000001:0.0:1784949179.829582:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:1.0:1784949179.829588:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000473c8c31 x1871651289679616 msgsize 224 00000020:00000001:0.0:1784949179.829589:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949179.829597:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00100000:1.0:1784949179.829604:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:0.0:1784949179.829605:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949179.829615:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949179.829617:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949179.829620:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949179.829622:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949179.829625:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949179.829627:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949179.829630:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949179.829632:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949179.829639:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:1.0:1784949179.829644:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000010:0.0:1784949179.829646:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002f80d253. 00000800:00000001:1.0:1784949179.829652:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949179.829653:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949179.829664:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949179.829668:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949179.829672:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289679488 00010000:00000001:0.0:1784949179.829676:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949179.829683:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000200:0.0:1784949179.829685:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b4879340 x1871651289679488/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:85/0 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949179.829690:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289679616 02000000:00000001:3.0:1784949179.829696:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949179.829701:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949179.829706:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949179.829710:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:3.0:1784949179.829714:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949179.829715:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949179.829723:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289679616 00000100:00001000:0.0:1784949179.829726:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=51 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949179.829730:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949179.829734:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784949179.829735:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1784949179.829739:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784949179.829743:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000040:3.0:1784949179.829747:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:0.0:1784949179.829748:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:3.0:1784949179.829754:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:0.0:1784949179.829756:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:3.0:1784949179.829763:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:0.0:1784949179.829764:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949179.829774:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949179.829775:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:0.0:1784949179.829781:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:3.0:1784949179.829784:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000400:00000010:0.0:1784949179.829787:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000001:3.0:1784949179.829791:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784949179.829795:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289679488, offset 224 00000020:00000010:3.0:1784949179.829797:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000400:00000200:0.0:1784949179.829810:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:3.0:1784949179.829813:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 00000020:00000010:3.0:1784949179.829821:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006924478a. 00000400:00000200:0.0:1784949179.829823:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:3.0:1784949179.829831:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000200:0.0:1784949179.829840:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c680 00000100:00000040:3.0:1784949179.829844:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949179.829849:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:0.0:1784949179.829850:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9125 [1] + 224 00000100:00000001:3.0:1784949179.829854:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949179.829861:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949179.829864:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949179.829869:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:0.0:1784949179.829876:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949179.829877:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949179.829882:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949179.829890:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949179.829901:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949179.829902:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949179.829943:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949179.829948:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949179.829961:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 770 00000100:00000001:2.0:1784949179.829966:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949179.829972:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949179.829973:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784949179.829978:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949179.829980:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949179.829981:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637801984 : -115544071749632 : ffff96e9cca5c000) 00000100:00000001:2.0:1784949179.829990:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949179.830000:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000473c8c31 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:85/0 lens 224/0 e 0 to 0 dl 1784949190 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949179.830016:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949179.830023:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949179.830027:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949179.830029:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949179.830030:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949179.830036:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000020:00000010:3.0:1784949179.830038:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:0.0:1784949179.830040:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000001:2.0:1784949179.830043:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784949179.830047:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:2.0:1784949179.830051:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949179.830052:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949179.830056:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949179.830058:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008c614621. 00000020:00000010:3.0:1784949179.830066:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000100:00000040:0.0:1784949179.830067:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:2.0:1784949179.830070:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00010000:00000001:0.0:1784949179.830077:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:2.0:1784949179.830078:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000020:00000001:0.0:1784949179.830083:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.830087:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1784949179.830090:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000473c8c31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289679616:12345-192.168.202.137@tcp:400: 00000020:00000001:0.0:1784949179.830094:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:2.0:1784949179.830115:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949179.830123:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b4879340 x1871651289679488/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:85/0 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949179.830126:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289679616 00000020:00000001:3.0:1784949179.830135:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949179.830145:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949179.830152:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949179.830162:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949179.830169:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000200:2.0:1784949179.830174:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949179.830177:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00100000:0.0:1784949179.830185:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b4879340 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289679488:12345-0@lo:400: Request processed in 658us (1187us total) trans 0 rc 0/0 00000020:00000001:3.0:1784949179.830187:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949179.830196:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949179.830202:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949179.830208:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949179.830218:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 794 00000100:00000001:2.0:1784949179.830220:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1784949179.830220:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:0.0:1784949179.830229:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949179.830231:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949179.830232:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949179.830236:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:3.0:1784949179.830240:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949179.830242:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784949179.830244:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 02000000:00000010:3.0:1784949179.830248:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d02b883a. 02000000:00000001:2.0:1784949179.830252:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:3.0:1784949179.830255:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949179.830256:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00000001:2.0:1784949179.830257:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949179.830264:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949179.830265:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949179.830270:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000001:3.0:1784949179.830275:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949179.830276:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949179.830279:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00010000:00000040:3.0:1784949179.830288:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289679616 00000100:00001000:2.0:1784949179.830291:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=51 v=5 (5 5 5 5) 00000020:00000040:0.0:1784949179.830295:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000001:3.0:1784949179.830295:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00001000:2.0:1784949179.830300:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=51 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949179.830307:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784949179.830308:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00010000:00000200:3.0:1784949179.830309:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000473c8c31 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:85/0 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949179.830312:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000002f80d253. 00000100:00000001:2.0:1784949179.830315:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949179.830324:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:2.0:1784949179.830327:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784949179.830330:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949179.830335:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949179.830343:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784949179.830344:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949179.830350:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949179.830356:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949179.830358:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bb684072. 00000100:00000040:2.0:1784949179.830361:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:3.0:1784949179.830363:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=46 v=5 (1 1 1 1) 00000400:00000010:0.0:1784949179.830370:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000017500e72. 00000100:00000001:3.0:1784949179.830381:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949179.830391:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949179.830392:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949179.830404:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949179.830404:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784949179.830410:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949179.830412:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949179.830417:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949179.830421:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949179.830424:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949179.830437:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949179.830448:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00100000:2.0:1784949179.830451:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289679488:0@lo:400: 00000100:00000200:3.0:1784949179.830457:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289679616, offset 224 00000100:00000001:2.0:1784949179.830481:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949179.830504:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949179.830509:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289679488/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949179.830519:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949179.830532:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949179.830536:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1784949179.830540:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000010:2.0:1784949179.830542:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001df3b40e. 02000000:00000001:2.0:1784949179.830552:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1784949179.830555:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3712:3712:242:4294967295] 192.168.202.137@tcp LPNI seq info [2381:2381:8:4294967295] 00000020:00000001:2.0:1784949179.830557:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949179.830562:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949179.830569:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949179.830572:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003a8a98bc. 02000000:00000001:2.0:1784949179.830581:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1784949179.830585:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:2.0:1784949179.830586:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949179.830591:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000800:00000200:3.0:1784949179.830599:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949179.830605:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000100:00000001:2.0:1784949179.830610:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000800:00000200:3.0:1784949179.830612:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949179.830614:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949179.830620:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949179.830625:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:2.0:1784949179.830626:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949179.830630:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949179.830634:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949179.830636:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949179.830641:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949179.830658:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949179.830665:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949179.830668:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949179.830670:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949179.830673:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949179.830677:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000473c8c31 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:85/0 lens 224/224 e 0 to 0 dl 1784949190 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949179.830688:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000473c8c31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289679616:12345-192.168.202.137@tcp:400: Request processed in 661us (1099us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949179.830699:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 770 00000100:00000040:3.0:1784949179.830702:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949179.830705:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949179.830708:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949179.830712:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006924478a. 00000020:00000010:3.0:1784949179.830717:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949179.830721:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00000020:00000040:3.0:1784949179.830728:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949179.830732:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949179.830859:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949179.830865:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:1.0:1784949179.830868:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000800:00000001:3.0:1784949179.830878:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949179.830881:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949179.830883:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949179.830890:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949179.830928:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949179.830937:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949179.830943:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c700 00000400:00000200:1.0:1784949179.830949:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000400:00000010:1.0:1784949179.830955:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00000001:1.0:1784949179.830960:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949179.830963:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949179.830968:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x912d [1] + 224 00000800:00000001:3.0:1784949179.830978:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949179.830994:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949179.830997:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949179.831002:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949179.831008:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949179.831014:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949179.831031:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949179.831058:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949179.831071:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949179.831077:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949179.831100:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949179.831105:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949179.831112:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949179.831114:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949179.831116:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000502cbfd9. 00000400:00000200:1.0:1784949179.831122:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949179.831123:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784949179.831126:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949179.831133:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949179.831144:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949179.831154:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949179.831156:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949179.831159:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949179.831161:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949179.831163:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949179.831164:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949179.831167:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949179.831173:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=46 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949179.831177:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=46 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949179.831180:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949179.831182:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949179.831184:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949179.831188:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949179.831197:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949179.831200:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949179.831202:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949179.831204:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949179.831206:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949179.831209:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949179.831216:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949179.831219:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949179.831221:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949179.831225:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949179.831234:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289679616:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949179.831239:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949179.831241:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949179.831243:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289679616/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949179.831250:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949179.831252:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949179.831254:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 02000000:00000001:1.0:1784949179.831258:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949179.831259:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949179.831261:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949179.831263:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949179.831265:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000057c7ab4. 02000000:00000001:1.0:1784949179.831267:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949179.831268:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949179.831270:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000100:00000001:1.0:1784949179.831275:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949179.831277:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949179.831278:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949179.831283:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949179.831285:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949179.831287:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949179.831290:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949179.872023:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306119211, at 4306119212 00000100:00000040:2.0:1784949179.872029:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306119211, at 4306119212 00000100:00000040:3.0:1784949179.872033:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306119211, at 4306119212 00000100:00000010:1.0:1784949179.872038:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:0.0:1784949179.872040:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306119211, at 4306119212 00000100:00000010:2.0:1784949179.872043:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:3.0:1784949179.872056:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:0.0:1784949179.872060:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00010000:00000001:2.0:1784949180.660048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949180.660061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949180.660071:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949180.660073:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949180.660141:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949180.660149:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949180.660154:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949180.852075:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949180.852096:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949180.852099:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949180.852109:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949180.852114:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949180.852120:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949180.852123:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949181.684072:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949181.684088:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949181.684121:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949181.684127:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949181.684131:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949181.684136:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949181.684140:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949181.684143:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949181.684146:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949181.684149:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949181.684302:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949181.684309:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949181.684314:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949181.876095:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949181.876112:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949181.876115:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949181.876121:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949181.876125:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949181.876130:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949181.876134:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949182.068007:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:1.0:1784949182.068019:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000800:00000200:1.0:1784949182.068025:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000800:00000200:2.0:1784949182.068171:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784949182.708059:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949182.708073:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949182.708086:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949182.708089:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949182.708219:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949182.708224:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949182.708227:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949182.900004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949182.900020:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949182.900023:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949182.900030:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949182.900034:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949182.900039:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949182.900044:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949183.732032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949183.732041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949183.732067:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949183.732070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949183.732172:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949183.732182:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949183.732189:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949183.924117:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949183.924132:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949183.924134:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949183.924138:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949183.924142:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949183.924146:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949183.924149:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949184.756054:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949184.756069:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949184.756084:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949184.756087:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949184.756223:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949184.756231:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949184.756236:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949184.948033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949184.948041:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949184.948063:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949159/real 1784949159] req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949184 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1784949184.948092:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949184.948096:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949184.948101:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000100:00000040:1.0:1784949184.948110:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1784949184.948111:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:2.0:1784949184.948118:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000100:00000001:1.0:1784949184.948129:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949184.948129:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949184.948136:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949184 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949184.948139:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005126bdfa. 00000020:00000040:1.0:1784949184.948150:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000200:2.0:1784949184.948159:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949184 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949184.948162:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949184.948170:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949184.948176:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949184.948180:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949184.948182:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.948184:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949184.948192:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949184.948197:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949184 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784949184.948199:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000b80ce7eb. 00000100:00000001:2.0:1784949184.948214:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949184.948217:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.948227:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949184.948231:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949184.948234:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.948244:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949184.948252:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949184.948257:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949184.948262:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949184.948267:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949184.948268:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949184 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949184.948284:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005126bdfa] to pc [ptlrpcd_00_03+3] req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.948292:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949184.948304:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.948312:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949184.948319:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.948323:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949184.948326:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949184.948330:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1784949184.948352:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949184.948358:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949184.948361:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00080000:1.0:1784949184.948368:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784949184.948371:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1784949184.948374:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784949184.948380:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1784949184.948384:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949184 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949184.948390:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784949184.948403:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11431 00000100:00100000:2.0:1784949184.948409:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289678464:192.168.202.137@tcp:8: 00000400:00000200:1.0:1784949184.948450:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:1.0:1784949184.948458:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11430 00000100:00000001:3.0:1784949184.948473:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949184.948482:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949184.948484:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:3.0:1784949184.948488:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1784949184.948494:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000040:3.0:1784949184.948501:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949184.948502:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949184.948511:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949184.948517:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949184.948521:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949184.948528:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949184.948531:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:3.0:1784949184.948539:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1784949184.948541:0:285579:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949184.948548:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949184.948549:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:1.0:1784949184.948557:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1784949184.948560:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784949184.948562:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000040:1.0:1784949184.948563:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:3.0:1784949184.948566:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949184.948573:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:0.0:1784949184.948577:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 02000000:00000001:3.0:1784949184.948577:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949184.948582:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00100000:3.0:1784949184.948590:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289679744:0@lo:400: 00000100:00000040:1.0:1784949184.948594:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000400:00000010:0.0:1784949184.948597:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004b465f54. 00000100:00000001:1.0:1784949184.948606:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949184.948608:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:0.0:1784949184.948612:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000040:1.0:1784949184.948614:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 02000000:00000001:3.0:1784949184.948618:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949184.948622:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:3.0:1784949184.948625:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949184.948631:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:0.0:1784949184.948632:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001e82affc. 02000000:00000001:3.0:1784949184.948633:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949184.948645:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000009f047fe3. 00000100:00080000:1.0:1784949184.948646:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000200:0.0:1784949184.948648:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 02000000:00000001:3.0:1784949184.948657:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949184.948663:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:1.0:1784949184.948670:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949184.948671:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fe908aa6. 00000100:00000001:2.0:1784949184.948674:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949184.948679:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1784949184.948681:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:2.0:1784949184.948686:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1784949184.948688:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 02000000:00000001:1.0:1784949184.948693:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949184.948695:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:3.0:1784949184.948701:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289679744, portal 25 00000100:00000040:2.0:1784949184.948702:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289678464/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949184 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949184.948706:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.948712:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784949184.948714:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949184.948717:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1784949184.948721:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637815424 : -115544071736192 : ffff96e9cca5f480) 00000100:00000001:1.0:1784949184.948726:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.948734:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000010:1.0:1784949184.948739:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000029a32fda. 00000100:00000040:3.0:1784949184.948740:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949184.948753:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000020:00000040:1.0:1784949184.948754:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=4 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:2.0:1784949184.948755:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949184.948767:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1784949184.948769:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000010:2.0:1784949184.948776:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006926284c. 00000100:00000001:3.0:1784949184.948777:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949184.948778:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.948779:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3713:3713:242:4294967295] 1.2.3.4@tcp LPNI seq info [112:112:8:4294967295] 00000100:00000001:1.0:1784949184.948791:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949184.948792:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949184.948792:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784949184.948802:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1784949184.948803:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000010:3.0:1784949184.948805:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 02000000:00000001:1.0:1784949184.948808:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949184.948810:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:0.0:1784949184.948814:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000200:3.0:1784949184.948815:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289679744, offset 0 02000000:00000010:1.0:1784949184.948822:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006e175381. 00000020:00000001:2.0:1784949184.948823:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949184.948832:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 0000000078a163a8. 00000100:00000001:1.0:1784949184.948834:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949184.948840:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784949184.948844:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949184.948851:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:1.0:1784949184.948857:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1784949184.948857:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 00000400:00000200:0.0:1784949184.948862:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:2.0:1784949184.948881:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949184.948884:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949184.948887:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949184.948891:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949184.948893:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:2.0:1784949184.948898:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949184.948901:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949184.948956:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:0.0:1784949184.948960:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000635133da. 00000100:00000040:1.0:1784949184.948961:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029a32fda] to pc [ptlrpcd_rcv+-1] req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.948963:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949184.948969:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c780 00000100:00000001:2.0:1784949184.948975:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949184.948982:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949184.948988:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:3.0:1784949184.948995:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8739 [2] + 0 00000800:00000200:0.0:1784949184.949002:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 32 secs later) 00000400:00000010:3.0:1784949184.949010:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d5071549. 00000400:00000200:3.0:1784949184.949024:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000532cd756 00000800:00000200:0.1:1784949184.949025:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:0.1:1784949184.949033:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1784949184.949039:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000400:00000200:3.0:1784949184.949042:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000010:0.0:1784949184.949044:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000635133da. 00000100:00000001:1.0:1784949184.949055:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.949056:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949184.949056:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:1.0:1784949184.949068:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1784949184.949074:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000532cd756 00000400:00000200:0.0:1784949184.949076:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000001:1.0:1784949184.949084:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000010:3.0:1784949184.949085:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000532cd756. 00000100:00000001:2.0:1784949184.949092:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784949184.949093:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cd114eaa. 00000100:00000001:3.0:1784949184.949098:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784949184.949099:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949184.949100:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000020:00000040:1.0:1784949184.949102:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000200:3.0:1784949184.949107:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:2.0:1784949184.949108:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949184.949111:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1784949184.949116:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 02000000:00000001:1.0:1784949184.949119:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1784949184.949121:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000569fbc5c x1871651289679744 msgsize 224 00000100:00000040:2.0:1784949184.949123:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949184.949127:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.949136:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000010:1.0:1784949184.949140:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e4c24e27. 00000100:00100000:3.0:1784949184.949149:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:1.0:1784949184.949153:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949184.949159:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784949184.949162:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:1.0:1784949184.949172:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784949184.949178:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000001:2.0:1784949184.949180:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949184.949181:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949184.949189:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949184.949200:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784949184.949200:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000400:00000200:0.0:1784949184.949203:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 02000000:00000001:2.0:1784949184.949209:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949184.949210:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1784949184.949211:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 00000100:00000001:1.0:1784949184.949217:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949184.949221:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949184.949225:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:0.0:1784949184.949226:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1784949184.949231:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd114eaa] to pc [ptlrpcd_00_00+0] req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949184.949233:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289679872:1.2.3.4@tcp:8: 00000400:00000001:0.0:1784949184.949235:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:3.0:1784949184.949241:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949184.949244:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000100:00000001:2.0:1784949184.949256:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:0.0:1784949184.949257:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:2.0:1784949184.949265:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.949270:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:2.0:1784949184.949276:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.949288:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:2.0:1784949184.949288:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.949300:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949184.949303:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004b465f54. 00000100:00000001:3.0:1784949184.949306:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949184.949307:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949184.949313:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949184.949316:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949184.949319:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.949321:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949184.949323:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:2.0:1784949184.949327:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784949184.949329:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11461) 00000100:00000001:2.0:1784949184.949335:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.949337:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000040:1.0:1784949184.949342:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11461) 00000100:00000001:2.0:1784949184.949345:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.949358:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:3.0:1784949184.949369:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949184.949370:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949184.949375:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949184.949376:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949184.949378:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949184.949386:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949184.949387:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949184.949391:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949184.949398:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949184.949403:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306124331 00000100:00000001:1.0:1784949184.949444:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:3.0:1784949184.949448:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:1.0:1784949184.949449:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949184.949459:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:1.0:1784949184.949460:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949184.949467:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1784949184.949468:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784949184.949469:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289679744 02000000:00000001:0.0:1784949184.949481:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:1.0:1784949184.949482:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949184.949484:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306124331 00000100:00000001:0.0:1784949184.949489:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000010:3.0:1784949184.949495:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000100:00100000:1.0:1784949184.949496:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289680000:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949184.949497:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949184.949509:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949184.949510:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949184.949512:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949184.949517:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949184.949521:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306124331 00000100:00100000:0.0:1784949184.949523:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289679744 02000000:00000001:1.0:1784949184.949523:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949184.949530:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000020:00000001:0.0:1784949184.949533:0:342756:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784949184.949534:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949184.949537:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000040:0.0:1784949184.949541:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 02000000:00000010:1.0:1784949184.949544:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 00000020:00000001:0.0:1784949184.949547:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1784949184.949552:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949184.949554:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949184.949555:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306124331 00000400:00000010:1.0:1784949184.949562:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000502cbfd9. 00000020:00000001:0.0:1784949184.949565:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784949184.949569:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003e9ea21c. 00000020:00000001:0.0:1784949184.949572:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000010:1.0:1784949184.949576:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000001:3.0:1784949184.949578:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949184.949583:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:1.0:1784949184.949586:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289680000, portal 10 00000100:00000001:3.0:1784949184.949589:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949184.949593:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:1.0:1784949184.949593:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949184.949596:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949184.949600:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637804672 : -115544071746944 : ffff96e9cca5ca80) 00000100:00000001:0.0:1784949184.949603:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949184.949607:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949184.949611:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:3.0:1784949184.949613:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949184.949617:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949184.949624:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:3.0:1784949184.949626:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949184.949633:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949184.949636:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:0.0:1784949184.949641:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000001:3.0:1784949184.949641:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949184.949649:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784949184.949650:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000010:0.0:1784949184.949656:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949184.949657:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949184.949659:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000100:00000001:3.0:1784949184.949670:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949184.949671:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289680000, offset 0 00000100:00000040:0.0:1784949184.949677:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949184.949685:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949184.949688:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:1.0:1784949184.949694:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949184.949695:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949184.949709:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949184.949712:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949184.949718:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949184.949724:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949184.949728:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949184.949731:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949184.949735:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3714:3714:242:4294967295] 192.168.202.137@tcp LPNI seq info [2382:2382:8:4294967295] 00000100:00000001:0.0:1784949184.949742:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949184.949755:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949184.949765:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949184.949771:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008d5370ec. 00000100:00000001:0.0:1784949184.949780:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000200:1.0:1784949184.949782:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949184.949790:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949184.949793:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949184.949799:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:1.0:1784949184.949802:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:0.0:1784949184.949809:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 795 00000800:00000200:1.0:1784949184.949811:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008d5370ec type 1, nob 320 niov 1 nkiov 1 00000100:00000040:0.0:1784949184.949823:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949184.949823:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.949831:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949184.949834:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742037584 : -115538967514032 : ffff96eafce23450) 00000100:00000001:1.0:1784949184.949841:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949184.949849:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000569fbc5c x1871651289679744/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:90/0 lens 224/0 e 0 to 0 dl 1784949195 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949184.949850:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2208 usecs (3 jiffies) 00000100:00000001:1.0:1784949184.949860:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306124292 00000100:00000001:1.0:1784949184.949870:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306124331 00000100:00000001:1.0:1784949184.949878:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306124292 00000100:00000001:0.0:1784949184.949879:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949184.949885:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306124331 00000020:00000010:0.0:1784949184.949889:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000530e1e90. 00000100:00000001:1.0:1784949184.949890:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306124292 00000100:00000001:0.0:1784949184.949902:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949184.949940:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306124331 00000100:00000001:1.0:1784949184.949944:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306124293 00000100:00000001:1.0:1784949184.949947:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306124331 00000100:00000001:1.0:1784949184.949957:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949184.949958:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000530e1e90. 00000100:00000001:1.0:1784949184.949967:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949184.949972:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949184.949974:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000569fbc5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289679744:12345-0@lo:400: 00000100:00000001:1.0:1784949184.949976:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949184.949988:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289679744 00000100:00000001:1.0:1784949184.949991:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784949184.949994:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949184.949996:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:0.0:1784949184.950004:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949184.950004:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949184.950012:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784949184.950013:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949184.950016:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949184.950019:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949184.950024:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949184.950025:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949184.950034:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949184.950037:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949184.950041:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949184.950043:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949184.950047:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949184.950050:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949184.950055:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949184.950059:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949184.950066:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000014f50db5. 02000000:00000001:0.0:1784949184.950070:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949184.950073:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949184.950076:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949184.950080:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289679744 00010000:00000001:0.0:1784949184.950084:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949184.950090:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000569fbc5c x1871651289679744/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:90/0 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949184.950108:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1784949184.950110:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000800:00000010:3.0:1784949184.950117:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008d5370ec. 00010000:00000001:0.0:1784949184.950118:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949184.950129:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=56 v=5 (1 1 1 1) 00000400:00000200:3.0:1784949184.950130:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949184.950142:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949184.950143:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1784949184.950151:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:0.0:1784949184.950153:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949184.950161:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:3.0:1784949184.950162:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000800:00000001:1.0:1784949184.950164:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949184.950166:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:3.0:1784949184.950169:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 02000000:00000001:0.0:1784949184.950173:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949184.950176:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949184.950179:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949184.950180:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:1.0:1784949184.950184:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949184.950193:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000200:3.0:1784949184.950194:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949184.950203:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000400:00000200:1.0:1784949184.950208:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:0.0:1784949184.950215:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289679744, offset 224 00000400:00000200:1.0:1784949184.950235:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643c880 00000100:00000001:3.0:1784949184.950236:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949184.950237:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:3.0:1784949184.950248:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949184.950261:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 173744 00000400:00000200:0.0:1784949184.950264:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:1.0:1784949184.950285:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949184.950286:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949184.950294:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c780 00000100:00000001:3.0:1784949184.950296:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784949184.950301:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949184.950308:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949184.950309:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9145 [1] + 224 00000400:00000200:1.0:1784949184.950320:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949184.950323:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949184.950330:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:0.0:1784949184.950336:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:1.0:1784949184.950337:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949184.950348:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:1.0:1784949184.950349:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007a42e32e. 00000100:00000040:1.0:1784949184.950356:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000007a42e32e x1871651289680000 msgsize 224 00000100:00000200:0.0:1784949184.950359:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949184.950375:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:0.0:1784949184.950390:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949184.950399:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:1.0:1784949184.950412:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949184.950417:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949184.950440:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949184.950458:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949184.950463:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784949184.950470:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949184.950473:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949184.950480:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949184.950485:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949184.950487:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784949184.950490:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949184.950493:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784949184.950495:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949184.950501:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:3.0:1784949184.950502:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949184.950508:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1784949184.950511:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000569fbc5c x1871651289679744/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:90/0 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949184.950514:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fe908aa6. 00000400:00000200:3.0:1784949184.950530:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000100:00100000:0.0:1784949184.950537:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000569fbc5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289679744:12345-0@lo:400: Request processed in 652us (1408us total) trans 0 rc 0/0 00000400:00000010:3.0:1784949184.950538:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:2.0:1784949184.950539:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949184.950549:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289680000 00000100:00000001:3.0:1784949184.950550:0:337122:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1784949184.950556:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949184.950563:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:3.0:1784949184.950565:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949184.950566:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 795 00000100:00000001:2.0:1784949184.950567:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949184.950581:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 02000000:00000001:2.0:1784949184.950582:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949184.950592:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:3.0:1784949184.950593:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949184.950595:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289680000 00000100:00000001:0.0:1784949184.950601:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:2.0:1784949184.950603:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949184.950610:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000010:0.0:1784949184.950613:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000020:00000001:2.0:1784949184.950618:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949184.950628:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1784949184.950629:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000010:0.0:1784949184.950630:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949184.950633:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949184.950643:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000010:0.0:1784949184.950644:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000100:00000001:3.0:1784949184.950644:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949184.950651:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:2.0:1784949184.950655:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949184.950657:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:0.0:1784949184.950658:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949184.950664:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.950665:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:0.0:1784949184.950668:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:3.0:1784949184.950674:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949184.950675:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000014f50db5. 00000020:00000010:2.0:1784949184.950676:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:2.0:1784949184.950688:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00001000:3.0:1784949184.950688:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=56 v=5 (5 5 5 5) 02000000:00000001:0.0:1784949184.950689:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949184.950697:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784949184.950700:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00001000:3.0:1784949184.950704:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=56 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949184.950708:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000010:2.0:1784949184.950713:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001df3b40e. 00000100:00000001:3.0:1784949184.950714:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:0.0:1784949184.950720:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d163afc3. 00000100:00000001:3.0:1784949184.950720:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949184.950724:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000059fdf11b. 00010000:00000001:3.0:1784949184.950729:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949184.950735:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949184.950740:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bed828fe. 00000020:00000010:2.0:1784949184.950741:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784949184.950742:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949184.950759:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949184.950765:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949184.950772:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949184.950773:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949184.950779:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949184.950781:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949184.950784:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949184.950786:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949184.950797:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.950798:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949184.950804:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949184.950809:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949184.950821:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289679744:0@lo:400: 00000100:00000001:3.0:1784949184.950827:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949184.950830:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949184.950832:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005126bdfa x1871651289679744/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.950837:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949184.950841:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949184.950846:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949184.950848:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000009f047fe3. 02000000:00000001:3.0:1784949184.950854:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949184.950857:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:2.0:1784949184.950858:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000040:3.0:1784949184.950859:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949184.950864:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000001:3.0:1784949184.950867:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949184.950872:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000b80ce7eb. 00000100:00100000:2.0:1784949184.950879:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 771 02000000:00000001:3.0:1784949184.950882:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1784949184.950885:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:3.0:1784949184.950888:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1784949184.950894:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637810944 : -115544071740672 : ffff96e9cca5e300) 00000100:00000010:3.0:1784949184.950895:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005126bdfa. 00000100:00000040:2.0:1784949184.950940:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000007a42e32e x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:90/0 lens 224/0 e 0 to 0 dl 1784949195 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949184.950950:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949184.950957:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.950958:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949184.950961:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784949184.950966:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:3.0:1784949184.950970:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949184.950972:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949184.950977:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949184.950979:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000001:3.0:1784949184.950989:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949184.950996:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000007a42e32e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289680000:12345-192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949184.950997:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949184.951009:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289680000 00000020:00000001:2.0:1784949184.951013:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949184.951017:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949184.951021:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949184.951024:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949184.951027:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949184.951030:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949184.951034:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949184.951037:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949184.951038:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949184.951041:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949184.951044:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949184.951048:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949184.951051:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949184.951055:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000511cd0b8. 02000000:00000001:2.0:1784949184.951058:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.951060:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949184.951063:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949184.951066:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289680000 00010000:00000001:2.0:1784949184.951069:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949184.951073:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000007a42e32e x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:90/0 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949184.951084:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949184.951086:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949184.951089:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=51 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949184.951094:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949184.951097:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949184.951101:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949184.951104:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949184.951106:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.951109:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949184.951112:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949184.951116:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000200:2.0:1784949184.951120:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289680000, offset 224 00000400:00000200:2.0:1784949184.951126:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949184.951137:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949184.951143:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3715:3715:242:4294967295] 192.168.202.137@tcp LPNI seq info [2383:2383:8:4294967295] 00000400:00000200:2.0:1784949184.951155:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949184.951161:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949184.951165:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007a3f8609. 00000800:00000200:2.0:1784949184.951171:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949184.951178:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949184.951182:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007a3f8609 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949184.951202:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949184.951206:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949184.951209:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949184.951211:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949184.951214:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949184.951218:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007a42e32e x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:90/0 lens 224/224 e 0 to 0 dl 1784949195 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949184.951229:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000007a42e32e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289680000:12345-192.168.202.137@tcp:400: Request processed in 271us (871us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949184.951238:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 771 00000100:00000040:2.0:1784949184.951241:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949184.951244:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949184.951247:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949184.951250:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000059fdf11b. 00000020:00000010:2.0:1784949184.951256:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949184.951260:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001df3b40e. 00000020:00000040:2.0:1784949184.951264:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949184.951267:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949184.951456:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949184.951463:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007a3f8609. 00000400:00000200:1.0:1784949184.951469:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949184.951477:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949184.951482:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:1.0:1784949184.951485:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000100:00000001:1.0:1784949184.951489:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949184.951492:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949184.951500:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949184.951508:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949184.951511:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949184.951515:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949184.951521:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949184.951533:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643c880 00000400:00000200:1.0:1784949184.951542:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x914d [1] + 224 00000800:00000001:1.0:1784949184.951549:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949184.951557:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949184.951559:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949184.951563:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949184.951567:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949184.951572:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949184.951585:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949184.951612:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949184.951618:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949184.951621:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.951720:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949184.951726:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949184.951734:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949184.951736:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949184.951739:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000502cbfd9. 00000400:00000200:2.0:1784949184.951747:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:2.0:1784949184.951749:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:2.0:1784949184.951753:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949184.951760:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949184.951774:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.951784:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949184.951786:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.951789:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949184.951791:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949184.951793:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949184.951795:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949184.951798:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949184.951804:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=51 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949184.951808:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=51 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949184.951811:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949184.951813:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949184.951815:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949184.951819:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949184.951829:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.951832:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949184.951834:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949184.951837:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949184.951873:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949184.951878:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.951887:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949184.951890:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949184.951892:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949184.951895:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949184.951934:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289680000:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949184.951941:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.951943:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949184.951946:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000cd114eaa x1871651289680000/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949200 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949184.951954:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949184.951957:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949184.951959:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 02000000:00000001:2.0:1784949184.951965:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949184.951967:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949184.951969:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949184.951972:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949184.951974:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e4c24e27. 02000000:00000001:2.0:1784949184.951977:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949184.951978:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949184.951980:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000cd114eaa. 00000100:00000001:2.0:1784949184.951983:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949184.951985:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.951987:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949184.951992:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949184.951994:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949184.951996:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949184.951999:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949184.992024:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306124331, at 4306124332 00000100:00000040:0.0:1784949184.992031:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306124331, at 4306124332 00000100:00000010:3.0:1784949184.992039:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003e9ea21c. 00000100:00000040:2.0:1784949184.992044:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306124331, at 4306124332 00000100:00000010:0.0:1784949184.992047:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:1.0:1784949184.992050:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306124331, at 4306124332 00000100:00000010:2.0:1784949184.992062:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:1.0:1784949184.992067:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00010000:00000001:2.0:1784949185.780046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949185.780058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949185.780068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949185.780070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949185.780218:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949185.780225:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949185.780230:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949185.972051:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949185.972059:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949185.972078:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949184/real 0] req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949185 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949185.972098:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.972102:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949185.972105:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949185.972109:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949185 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949185.972119:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.972130:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949185.972133:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.972138:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949185.972143:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949185.972146:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949185.972150:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949185 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949185.972160:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949185.972165:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.972167:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949185.972168:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.972171:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949185.972174:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949185.972179:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949185.972207:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:2.0:1784949185.972211:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949185.972216:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949185 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949185.972235:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289679872:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949185.972241:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.972245:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949185.972248:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000029a32fda x1871651289679872/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949185 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949185.972256:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1784949185.972261:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949185.972263:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784949185.972266:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949185.972272:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000006e175381. 02000000:00000001:2.0:1784949185.972280:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949185.972283:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949185.972285:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000029a32fda. 00000100:00000001:2.0:1784949185.972291:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949185.972294:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.972296:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.972301:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949185.972303:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.972307:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949185.972309:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949185.972323:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949185.972334:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949185.972343:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000029a32fda. 00000020:00000040:1.0:1784949185.972350:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949185.972356:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949185.972360:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949185.972368:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949185.972377:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000bc928ba2. 00000100:00000001:1.0:1784949185.972387:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949185.972396:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949185.972453:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949185.972457:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949185.972464:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029a32fda] to pc [ptlrpcd_00_01+1] req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949185.972509:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949185.972512:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949185.972517:0:285579:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784949185.972520:0:285579:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784949185.972523:0:285579:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784949185.972529:0:285579:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784949185.972536:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11431 00000100:00000001:2.0:1784949185.972539:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949185.972545:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949185.972548:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1784949185.972553:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000040:2.0:1784949185.972561:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1784949185.972563:0:285579:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11456 00000100:00000001:3.0:1784949185.972569:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949185.972579:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.972581:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949185.972587:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949185.972589:0:285579:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 02000000:00000001:2.0:1784949185.972597:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784949185.972603:0:285579:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 02000000:00000001:2.0:1784949185.972605:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1784949185.972608:0:285579:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1784949185.972611:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949185.972618:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00100000:2.0:1784949185.972622:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289680128:0@lo:400: 00000100:00000001:1.0:1784949185.972626:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949185.972634:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949185.972637:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949185.972643:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949185.972645:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1784949185.972649:0:285579:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1784949185.972652:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949185.972654:0:285579:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:2.0:1784949185.972659:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1784949185.972661:0:285579:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:0.0:1784949185.972669:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000040:1.0:1784949185.972669:0:285579:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000010:2.0:1784949185.972670:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001df3b40e. 00000020:00000001:1.0:1784949185.972682:0:285579:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:2.0:1784949185.972683:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949185.972688:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000020:00000001:1.0:1784949185.972695:0:285579:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:2.0:1784949185.972701:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000400:00000010:0.0:1784949185.972706:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004b465f54. 00000100:00000040:1.0:1784949185.972707:0:285579:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949185.972715:0:285579:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1784949185.972716:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000010:0.0:1784949185.972723:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000040:1.0:1784949185.972725:0:285579:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000200:2.0:1784949185.972730:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289680128, portal 25 00000100:00000001:1.0:1784949185.972736:0:285579:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784949185.972737:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949185.972743:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637809152 : -115544071742464 : ffff96e9cca5dc00) 00000020:00000040:1.0:1784949185.972746:0:285579:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:0.0:1784949185.972747:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e2f3909c. 00000100:00000040:2.0:1784949185.972754:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1784949185.972759:0:285579:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000400:00000200:0.0:1784949185.972773:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00080000:1.0:1784949185.972776:0:285579:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:1.0:1784949185.972790:0:285579:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.972796:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949185.972797:0:285579:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1784949185.972806:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784949185.972811:0:285579:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949185.972812:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000400:00000010:2.0:1784949185.972816:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 02000000:00000001:1.0:1784949185.972823:0:285579:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949185.972828:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289680128, offset 0 00000100:00000001:1.0:1784949185.972830:0:285579:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:0.0:1784949185.972836:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949185.972837:0:285579:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949185.972848:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000010:1.0:1784949185.972852:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cd114eaa. 00000400:00000200:0.0:1784949185.972854:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000040:1.0:1784949185.972862:0:285579:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:0.0:1784949185.972869:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1784949185.972874:0:285579:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1784949185.972879:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:1.0:1784949185.972881:0:285579:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949185.972887:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1784949185.972891:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3716:3716:242:4294967295] 1.2.3.4@tcp LPNI seq info [113:113:8:4294967295] 02000000:00000001:1.0:1784949185.972894:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949185.972899:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949185.972945:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000b91d4ef5. 00000100:00000001:1.0:1784949185.972951:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949185.972952:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00100000:1.0:1784949185.972971:0:285579:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1784949185.972982:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:1.0:1784949185.973007:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1784949185.973009:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:1.0:1784949185.973012:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949185.973023:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:0.0:1784949185.973024:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000635133da. 00000100:00000040:1.0:1784949185.973027:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd114eaa] to pc [ptlrpcd_rcv+-1] req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784949185.973048:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:2.0:1784949185.973057:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643c900 00000800:00000200:0.0:1784949185.973063:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 30 secs later) 00000400:00000200:2.0:1784949185.973073:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8759 [2] + 0 00000100:00000001:1.0:1784949185.973076:0:285579:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1784949185.973083:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:1.0:1784949185.973090:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1784949185.973092:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000800:00000200:0.1:1784949185.973093:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 30 secs later) 00000100:00000001:1.0:1784949185.973109:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:2.0:1784949185.973110:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000449b49f2 00000800:00000100:0.0:1784949185.973117:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000100:00000010:1.0:1784949185.973120:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005126bdfa. 00000800:00000010:0.0:1784949185.973123:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000635133da. 00000400:00000200:2.0:1784949185.973129:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1784949185.973130:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:0.0:1784949185.973134:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1784949185.973139:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949185.973142:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1784949185.973146:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:1.0:1784949185.973149:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949185.973150:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784949185.973157:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949185.973158:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784949185.973159:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949185.973159:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000449b49f2 02000000:00000010:1.0:1784949185.973168:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c158c1d0. 00000400:00000010:2.0:1784949185.973171:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000449b49f2. 00000100:00000001:1.0:1784949185.973178:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949185.973178:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949185.973183:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:0.0:1784949185.973186:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000200:2.0:1784949185.973190:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784949185.973195:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784949185.973202:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000059bd87f4 x1871651289680128 msgsize 224 00000400:00000200:0.0:1784949185.973204:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:0.0:1784949185.973225:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00100000:2.0:1784949185.973236:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784949185.973241:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1784949185.973248:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949185.973249:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949185.973251:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:1.0:1784949185.973254:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949185.973262:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949185.973263:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784949185.973265:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 02000000:00000001:3.0:1784949185.973271:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949185.973274:0:337120:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:3.0:1784949185.973279:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949185.973281:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005126bdfa] to pc [ptlrpcd_00_02+2] req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949185.973282:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000200:2.0:1784949185.973291:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000100:00100000:3.0:1784949185.973294:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289680256:192.168.202.137@tcp:8: 00000400:00000010:2.0:1784949185.973299:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:2.0:1784949185.973310:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949185.973313:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000001:0.0:1784949185.973314:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949185.973323:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1784949185.973323:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000200:2.0:1784949185.973324:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949185.973331:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949185.973333:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949185.973343:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 02000000:00000001:3.0:1784949185.973344:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949185.973351:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.973353:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949185.973355:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 02000000:00000010:3.0:1784949185.973357:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000008618f1b2. 00000100:00000040:2.0:1784949185.973365:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949185.973366:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11463) 02000000:00000001:3.0:1784949185.973367:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949185.973372:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:3.0:1784949185.973383:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fe908aa6. 00000400:00000010:0.0:1784949185.973385:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004b465f54. 00000100:00000001:1.0:1784949185.973391:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949185.973400:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00000001:1.0:1784949185.973401:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949185.973401:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949185.973430:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784949185.973434:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:2.0:1784949185.973439:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784949185.973441:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949185.973442:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289680256, portal 4 00000400:00000200:0.0:1784949185.973447:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:2.0:1784949185.973452:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949185.973453:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949185.973460:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637804672 : -115544071746944 : ffff96e9cca5ca80) 00000100:00000001:2.0:1784949185.973468:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306125355 00000400:00000200:0.0:1784949185.973469:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:1.0:1784949185.973481:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:2.0:1784949185.973484:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000addba683. 00000100:00000001:1.0:1784949185.973485:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949185.973495:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949185.973500:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949185.973502:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1784949185.973505:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949185.973510:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949185.973514:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949185.973518:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306125355 00000100:00100000:1.0:1784949185.973519:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289680384:192.168.202.137@tcp:400: 00000100:00100000:0.0:1784949185.973523:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289680128 00000100:00000010:2.0:1784949185.973531:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000094130822. 02000000:00000001:0.0:1784949185.973533:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949185.973540:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949185.973543:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949185.973545:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1784949185.973548:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949185.973551:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949185.973555:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.973558:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306125355 02000000:00000001:0.0:1784949185.973567:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949185.973568:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949185.973569:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1784949185.973569:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 00000100:00100000:0.0:1784949185.973581:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289680128 00000100:00000001:2.0:1784949185.973582:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 02000000:00000010:1.0:1784949185.973583:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 00000100:00000040:3.0:1784949185.973591:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949185.973592:0:342757:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784949185.973594:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.973596:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306125355 00000020:00000040:0.0:1784949185.973599:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000400:00000010:3.0:1784949185.973601:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000020:00000001:0.0:1784949185.973608:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:2.0:1784949185.973609:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f59314ac. 00000400:00000010:1.0:1784949185.973610:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000768f23df. 00000100:00000200:3.0:1784949185.973610:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289680256, offset 0 00000020:00000040:0.0:1784949185.973617:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784949185.973626:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949185.973626:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000010:1.0:1784949185.973629:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000001:2.0:1784949185.973637:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949185.973638:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949185.973639:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949185.973646:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.973652:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949185.973653:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:1.0:1784949185.973654:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289680384, portal 10 00000100:00000001:2.0:1784949185.973656:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949185.973663:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000010:0.0:1784949185.973665:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:1.0:1784949185.973666:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949185.973672:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949185.973676:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637815424 : -115544071736192 : ffff96e9cca5f480) 00000100:00000001:0.0:1784949185.973677:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949185.973678:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:3.0:1784949185.973679:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1784949185.973693:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:2.0:1784949185.973694:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949185.973695:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:1.0:1784949185.973701:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949185.973704:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949185.973713:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1784949185.973714:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:2.0:1784949185.973714:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.973727:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949185.973728:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949185.973731:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:0.0:1784949185.973735:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000040:1.0:1784949185.973748:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949185.973749:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3717:3717:242:4294967295] 192.168.202.137@tcp LPNI seq info [2384:2384:8:4294967295] 00000020:00000010:0.0:1784949185.973753:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000010:1.0:1784949185.973758:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00000200:1.0:1784949185.973767:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289680384, offset 0 00000100:00000001:2.0:1784949185.973775:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949185.973781:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949185.973782:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949185.973784:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949185.973795:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949185.973799:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949185.973807:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000200:3.0:1784949185.973808:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949185.973813:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949185.973818:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949185.973821:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000400:00000200:1.0:1784949185.973823:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949185.973834:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949185.973840:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949185.973842:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949185.973847:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949185.973856:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949185.973859:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3718:3718:241:4294967295] 192.168.202.137@tcp LPNI seq info [2385:2385:7:4294967295] 00000800:00000200:3.0:1784949185.973869:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784949185.973871:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949185.973878:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1784949185.973891:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949185.973901:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949185.973940:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949185.973950:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 796 00000100:00000001:3.0:1784949185.973950:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949185.973952:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949185.973963:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000040:0.0:1784949185.973965:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000800:00000010:1.0:1784949185.973967:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002ddfcc2d. 00000100:00000001:3.0:1784949185.973973:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306125356 00000100:00000001:0.0:1784949185.973976:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742025296 : -115538967526320 : ffff96eafce20450) 00000800:00000200:1.0:1784949185.973987:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949185.973989:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4306125355 00000100:00000040:0.0:1784949185.973994:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000059bd87f4 x1871651289680128/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:91/0 lens 224/0 e 0 to 0 dl 1784949196 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949185.974002:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306125356 00000800:00000200:1.0:1784949185.974008:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784949185.974018:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4306125355 00000800:00000200:1.0:1784949185.974020:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002ddfcc2d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949185.974035:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949185.974038:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306125356 00000020:00000010:0.0:1784949185.974043:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:1.0:1784949185.974048:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949185.974051:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949185.974052:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4306125355 00000100:00000040:1.0:1784949185.974059:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2210 usecs (3 jiffies) 00000020:00000010:0.0:1784949185.974062:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:1.0:1784949185.974069:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306125317 00000100:00000001:3.0:1784949185.974070:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306125356 00000100:00000001:1.0:1784949185.974077:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306125356 00000100:00100000:0.0:1784949185.974078:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000059bd87f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289680128:12345-0@lo:400: 00000100:00000001:1.0:1784949185.974088:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306125317 00000100:00000001:3.0:1784949185.974089:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4306125355 00000100:00000200:0.0:1784949185.974091:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289680128 00000100:00000001:1.0:1784949185.974098:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306125356 00000020:00000001:0.0:1784949185.974101:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949185.974105:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949185.974106:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306125317 00000020:00000001:0.0:1784949185.974110:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949185.974118:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306125356 00000020:00000001:0.0:1784949185.974119:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949185.974120:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949185.974130:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306125317 00000020:00000001:0.0:1784949185.974131:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784949185.974135:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949185.974140:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784949185.974140:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306125356 00000100:00000001:3.0:1784949185.974144:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949185.974145:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949185.974153:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949185.974156:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000010:2.0:1784949185.974157:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000100:00000001:1.0:1784949185.974160:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949185.974164:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949185.974164:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949185.974170:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949185.974171:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949185.974175:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949185.974175:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1784949185.974179:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949185.974184:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949185.974190:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784949185.974192:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949185.974193:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1784949185.974193:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949185.974197:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949185.974201:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:0.0:1784949185.974205:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949185.974207:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949185.974211:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000100:00000001:1.0:1784949185.974213:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949185.974213:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949185.974214:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949185.974221:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1784949185.974221:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 02000000:00000001:0.0:1784949185.974224:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949185.974230:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.974234:0:337114:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:0.0:1784949185.974235:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000014f50db5. 00000100:00000001:1.0:1784949185.974237:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949185.974245:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949185.974247:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949185.974249:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949185.974251:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949185.974262:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949185.974270:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:0.0:1784949185.974273:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289680128 00000800:00000001:3.0:1784949185.974281:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:0.0:1784949185.974284:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949185.974290:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000200:0.0:1784949185.974297:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000059bd87f4 x1871651289680128/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:91/0 lens 224/224 e 0 to 0 dl 1784949196 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949185.974302:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949210 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949185.974353:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949185.974353:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.974362:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1784949185.974364:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949185.974373:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:0.0:1784949185.974375:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=57 v=5 (1 1 1 1) 00000800:00000200:1.0:1784949185.974382:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949185.974387:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949185.974388:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1784949185.974389:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002ddfcc2d. 00000800:00000001:2.0:1784949185.974395:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949185.974402:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784949185.974403:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:2.0:1784949185.974404:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949185.974443:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000400:00000200:1.0:1784949185.974443:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949185.974444:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949185.974454:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949185.974460:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 02000000:00000001:0.0:1784949185.974462:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:1.0:1784949185.974469:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 02000000:00000001:0.0:1784949185.974471:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949185.974476:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949185.974479:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949185.974483:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ca00 00000100:00000001:0.0:1784949185.974488:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:1.0:1784949185.974494:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949185.974507:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:2.0:1784949185.974512:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 173968 00000400:00000200:3.0:1784949185.974515:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643c980 00000400:00000010:0.0:1784949185.974523:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000800:00000001:2.0:1784949185.974537:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784949185.974543:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289680128, offset 224 00000100:00000001:1.0:1784949185.974543:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000100:3.0:1784949185.974550:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289680256 offset 0 length 520: 4 00000100:00000040:1.0:1784949185.974556:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949185.974558:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949185.974563:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000bcd9abe2 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:2.0:1784949185.974571:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949185.974572:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949185.974574:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000200:2.0:1784949185.974595:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949185.974596:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949185.974601:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949185.974607:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949185.974609:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949185.974610:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:2.0:1784949185.974612:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000800:00000001:3.0:1784949185.974623:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784949185.974627:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064fd9c10. 00000800:00000001:3.0:1784949185.974630:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949185.974638:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000064fd9c10 x1871651289680384 msgsize 224 00000400:00000200:0.0:1784949185.974639:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643c900 00000800:00000001:3.0:1784949185.974641:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949185.974664:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:0.0:1784949185.974665:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9161 [1] + 224 00000400:00000200:0.0:1784949185.974678:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949185.974681:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949185.974689:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:2.0:1784949185.974698:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949185.974701:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:2.0:1784949185.974709:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949185.974712:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949185.974721:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949185.974728:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289680384 02000000:00000001:3.0:1784949185.974734:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949185.974741:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949185.974745:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949185.974747:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949185.974751:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949185.974761:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289680384 00000020:00000001:3.0:1784949185.974767:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949185.974772:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949185.974778:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949185.974783:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949185.974789:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949185.974793:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949185.974797:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949185.974802:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:3.0:1784949185.974806:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949185.974807:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784949185.974809:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000400:00000200:0.0:1784949185.974821:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000020:00000010:3.0:1784949185.974822:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dd0652fb. 00000400:00000010:0.0:1784949185.974829:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000020:00000010:3.0:1784949185.974832:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000004de9c75. 00000100:00000001:0.0:1784949185.974836:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000010:3.0:1784949185.974842:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949185.974843:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949185.974856:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.974859:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949185.974861:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:0.0:1784949185.974866:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784949185.974868:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949185.974869:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00010000:00000001:0.0:1784949185.974872:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949185.974876:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:0.0:1784949185.974878:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784949185.974880:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949185.974882:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949185.974885:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:2.0:1784949185.974887:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949185.974893:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000100:00000001:3.0:1784949185.974895:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000040:0.0:1784949185.974899:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000059bd87f4 x1871651289680128/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:91/0 lens 224/224 e 0 to 0 dl 1784949196 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949185.974944:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000200:2.0:1784949185.974949:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000100:00000001:3.0:1784949185.974950:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949185.974957:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000100:00000001:2.0:1784949185.974968:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949185.974970:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949185.974975:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000059bd87f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289680128:12345-0@lo:400: Request processed in 931us (1761us total) trans 0 rc 0/0 00000100:00000200:2.0:1784949185.974981:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949185.974988:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949185.974996:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949185.975007:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 796 00000100:00100000:3.0:1784949185.975012:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 772 00000100:00000040:0.0:1784949185.975023:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000040:3.0:1784949185.975025:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949185.975033:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:2.0:1784949185.975033:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949185.975034:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562016768 : -115544147534848 : ffff96e9c8215c00) 00000100:00000001:0.0:1784949185.975042:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:3.0:1784949185.975054:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000064fd9c10 x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:91/0 lens 224/0 e 0 to 0 dl 1784949196 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949185.975055:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00000001:2.0:1784949185.975060:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949185.975067:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949185.975068:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949185.975078:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:0.0:1784949185.975082:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 02000000:00000001:2.0:1784949185.975085:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949185.975090:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949185.975093:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:0.0:1784949185.975097:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784949185.975099:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949185.975099:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 02000000:00000001:0.0:1784949185.975108:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:2.0:1784949185.975110:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949185.975111:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1784949185.975113:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000014f50db5. 00000100:00001000:2.0:1784949185.975123:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=57 v=5 (5 5 5 5) 00000020:00000010:3.0:1784949185.975125:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 02000000:00000001:0.0:1784949185.975126:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949185.975133:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949185.975139:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=57 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949185.975142:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:3.0:1784949185.975150:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000064fd9c10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289680384:12345-192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949185.975154:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:0.0:1784949185.975156:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b85c9890. 00000100:00000001:2.0:1784949185.975160:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949185.975163:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289680384 00010000:00000001:2.0:1784949185.975169:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949185.975170:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:0.0:1784949185.975171:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b3c5fd98. 00010000:00000001:2.0:1784949185.975176:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949185.975180:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:2.0:1784949185.975183:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949185.975186:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949185.975192:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:2.0:1784949185.975195:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949185.975197:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949185.975208:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949185.975217:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949185.975218:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949185.975228:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949185.975229:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1784949185.975234:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:2.0:1784949185.975235:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949185.975240:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949185.975245:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949185.975248:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949185.975257:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949185.975262:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:2.0:1784949185.975266:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289680128:0@lo:400: 02000000:00000010:3.0:1784949185.975271:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003345e898. 00000100:00000001:2.0:1784949185.975273:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949185.975279:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.975281:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949185.975287:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949185.975289:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000029a32fda x1871651289680128/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949185.975294:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949185.975303:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289680384 00010000:00000001:3.0:1784949185.975312:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949185.975314:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949185.975319:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00010000:00000200:3.0:1784949185.975324:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000064fd9c10 x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:91/0 lens 224/224 e 0 to 0 dl 1784949196 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784949185.975325:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001df3b40e. 02000000:00000001:2.0:1784949185.975335:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949185.975340:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949185.975344:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00010000:00000001:3.0:1784949185.975350:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000001:2.0:1784949185.975351:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000001:3.0:1784949185.975358:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949185.975359:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000bc928ba2. 00000100:00001000:3.0:1784949185.975368:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=52 v=5 (1 1 1 1) 02000000:00000001:2.0:1784949185.975369:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949185.975374:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949185.975380:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000029a32fda. 00000100:00000001:3.0:1784949185.975381:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949185.975386:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:3.0:1784949185.975390:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949185.975391:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949185.975400:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784949185.975401:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949185.975437:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949185.975439:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949185.975443:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949185.975445:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1784949185.975452:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949185.975453:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949185.975468:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949185.975469:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000001:2.0:1784949185.975472:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949185.975479:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289680384, offset 224 00000400:00000200:3.0:1784949185.975488:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949185.975498:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949185.975505:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3719:3719:242:4294967295] 192.168.202.137@tcp LPNI seq info [2386:2386:8:4294967295] 00000400:00000200:3.0:1784949185.975517:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949185.975522:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949185.975527:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949185.975532:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949185.975538:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949185.975542:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949185.975559:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949185.975563:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949185.975566:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949185.975568:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949185.975570:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949185.975574:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064fd9c10 x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:91/0 lens 224/224 e 0 to 0 dl 1784949196 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949185.975586:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000064fd9c10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289680384:12345-192.168.202.137@tcp:400: Request processed in 496us (947us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949185.975595:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 772 00000100:00000040:3.0:1784949185.975598:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949185.975601:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949185.975603:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949185.975606:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000004de9c75. 00000020:00000010:3.0:1784949185.975611:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949185.975615:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dd0652fb. 00000020:00000040:3.0:1784949185.975619:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949185.975622:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949185.975742:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949185.975749:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:1.0:1784949185.975757:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949185.975767:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949185.975773:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000400:00000010:1.0:1784949185.975776:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000001:1.0:1784949185.975781:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784949185.975784:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949185.975785:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949185.975800:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949185.975806:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949185.975810:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949185.975819:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949185.975831:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ca00 00000400:00000200:3.0:1784949185.975838:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9175 [1] + 224 00000800:00000001:3.0:1784949185.975844:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949185.975850:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949185.975852:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949185.975857:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949185.975862:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949185.975867:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949185.975878:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949185.975900:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949185.975940:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949185.975945:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949185.975948:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949185.975954:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949185.975960:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949185.975963:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949185.975966:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000768f23df. 00000400:00000200:1.0:1784949185.975972:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:1.0:1784949185.975975:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000100:00000001:1.0:1784949185.975978:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949185.975984:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949185.975997:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949185.976007:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949185.976009:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949185.976011:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949185.976014:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949185.976016:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949185.976018:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949185.976021:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949185.976027:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=52 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949185.976031:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=52 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949185.976034:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949185.976036:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949185.976039:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949185.976043:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949185.976052:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949185.976056:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949185.976058:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949185.976060:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949185.976061:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949185.976065:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949185.976073:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949185.976076:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949185.976078:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949185.976081:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949185.976090:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289680384:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949185.976095:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949185.976097:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949185.976100:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005126bdfa x1871651289680384/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949201 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949185.976107:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949185.976109:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949185.976111:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 02000000:00000001:1.0:1784949185.976115:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949185.976117:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949185.976119:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949185.976121:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949185.976124:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c158c1d0. 02000000:00000001:1.0:1784949185.976127:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949185.976129:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949185.976130:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005126bdfa. 00000100:00000001:1.0:1784949185.976134:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949185.976135:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949185.976137:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949185.976142:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949185.976144:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949185.976147:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949185.976149:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949186.017024:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306125356, at 4306125357 00000100:00000040:1.0:1784949186.017030:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306125356, at 4306125357 00000100:00000010:0.0:1784949186.017039:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000addba683. 00000100:00000010:1.0:1784949186.017045:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000094130822. 00000100:00000040:2.0:1784949186.017047:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306125356, at 4306125357 00000100:00000040:3.0:1784949186.017054:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306125356, at 4306125357 00000100:00000010:2.0:1784949186.017062:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000010:3.0:1784949186.017069:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f59314ac. 00010000:00000001:2.0:1784949186.804031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949186.804043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949186.804054:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949186.804056:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949186.804178:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949186.804186:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949186.804191:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949187.827983:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949187.827991:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949187.827997:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949187.827998:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949187.828092:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949187.828097:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949187.828100:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949188.852073:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949188.852084:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949188.852095:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949188.852096:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949188.852196:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949188.852205:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949188.852212:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949189.876127:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949189.876143:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949189.876158:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949189.876162:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949189.876257:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949189.876265:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949189.876271:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949190.900105:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949190.900119:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949190.900130:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949190.900132:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949190.900287:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949190.900300:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949190.900308:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949191.092108:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949191.092127:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949191.092129:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949191.092130:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949191.092142:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949191.092144:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784949191.092147:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949191.092154:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949191.092158:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000100:00000001:3.0:1784949191.092159:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949191.092169:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949191.092177:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949191.092181:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949191.092190:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949191.092205:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000635133da. 00000100:00000001:0.0:1784949191.092214:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949191.092226:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949191.092241:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949191.092244:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949191.092251:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b08fd3] to pc [ptlrpcd_00_03+3] req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949191.092287:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949191.092290:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949191.092295:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949191.092298:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949191.092302:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949191.092306:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949191.092310:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949191.092312:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949191.092314:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949191.092320:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949191.092324:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d104522b. 00000100:00000001:0.0:1784949191.092331:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949191.092342:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949191.092349:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949191.092353:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949191.092353:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949191.092363:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.092385:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949191.092392:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.092392:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949191.092413:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_00+0] req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949191.092430:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949191.092449:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949191.092453:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949191.092459:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949191.092465:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949191.092467:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949191.092470:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949191.092479:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11468) 00000100:00100000:2.0:1784949191.092480:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289680512:0@lo:400: 00000100:00000001:3.0:1784949191.092493:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949191.092506:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949191.092508:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949191.092513:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949191.092515:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949191.092522:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949191.092522:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784949191.092535:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949191.092538:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949191.092549:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949191.092563:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:2.0:1784949191.092568:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001df3b40e. 00000100:00000001:3.0:1784949191.092569:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949191.092588:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949191.092591:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949191.092598:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949191.092600:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 02000000:00000001:3.0:1784949191.092608:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949191.092614:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00100000:3.0:1784949191.092619:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289680640:192.168.202.137@tcp:400: 00000100:00000200:2.0:1784949191.092627:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289680512, portal 25 00000100:00000001:3.0:1784949191.092628:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949191.092638:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949191.092641:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 02000000:00000001:3.0:1784949191.092641:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949191.092648:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949191.092664:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949191.092672:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000378c7327. 00000100:00000040:2.0:1784949191.092678:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949191.092678:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949191.092700:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c6713e8b. 00000100:00000001:2.0:1784949191.092702:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949191.092717:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949191.092719:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000400:00000010:2.0:1784949191.092740:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000200:3.0:1784949191.092741:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289680640, portal 10 00000100:00000001:3.0:1784949191.092751:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000200:2.0:1784949191.092752:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289680512, offset 0 00000100:00000001:3.0:1784949191.092759:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000400:00000200:2.0:1784949191.092768:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:3.0:1784949191.092777:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949191.092791:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949191.092796:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949191.092808:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:3.0:1784949191.092809:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000100:00000200:3.0:1784949191.092832:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289680640, offset 0 00000400:00000200:3.0:1784949191.092845:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949191.092850:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ca80 00000400:00000200:2.0:1784949191.092863:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x87f1 [2] + 0 00000400:00000010:2.0:1784949191.092869:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000034b95863. 00000400:00000200:3.0:1784949191.092873:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949191.092881:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949191.092882:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000095fae736 00000400:00000200:3.0:1784949191.092892:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949191.092898:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949191.092899:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949191.092914:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949191.092943:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949191.092949:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3720:3720:242:4294967295] 192.168.202.137@tcp LPNI seq info [2387:2387:8:4294967295] 00000400:00000200:3.0:1784949191.092969:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949191.092977:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949191.092985:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000095fae736 00000800:00000010:3.0:1784949191.092985:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000400:00000010:2.0:1784949191.092999:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000095fae736. 00000800:00000200:3.0:1784949191.093007:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949191.093013:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949191.093018:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949191.093024:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000092af50af x1871651289680512 msgsize 224 00000800:00000200:3.0:1784949191.093028:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949191.093035:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:2.0:1784949191.093062:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949191.093066:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949191.093074:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949191.093082:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2212 usecs (3 jiffies) 00000100:00000001:2.0:1784949191.093084:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949191.093088:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306130436 00000100:00000010:3.0:1784949191.093096:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003e9ea21c. 00000400:00000200:2.0:1784949191.093101:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000010:2.0:1784949191.093106:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:3.0:1784949191.093107:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949191.093128:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949191.093134:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306130436 00000100:00000200:2.0:1784949191.093143:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949191.093146:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000010:3.0:1784949191.093148:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000100:00100000:0.0:1784949191.093160:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289680512 00000100:00000001:3.0:1784949191.093168:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 02000000:00000001:0.0:1784949191.093170:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949191.093184:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949191.093189:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306130436 00000100:00000001:0.0:1784949191.093196:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949191.093210:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 02000000:00000001:0.0:1784949191.093211:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.093212:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1784949191.093231:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289680512 00000100:00000001:3.0:1784949191.093233:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000040:2.0:1784949191.093236:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949191.093244:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949191.093248:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306130436 00000020:00000040:0.0:1784949191.093254:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:3.0:1784949191.093270:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000020:00000001:0.0:1784949191.093271:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949191.093287:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949191.093289:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000040:0.0:1784949191.093290:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949191.093308:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1784949191.093309:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949191.093311:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949191.093325:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949191.093325:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.093330:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949191.093333:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949191.093338:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949191.093346:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949191.093348:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306130475 00000020:00000010:0.0:1784949191.093350:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:3.0:1784949191.093361:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949191.093391:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949191.093398:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949191.093401:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784949191.093407:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:3.0:1784949191.093408:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949191.093419:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.093421:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306130436 00000800:00000200:1.0:1784949191.093422:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1784949191.093426:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:3.0:1784949191.093428:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949191.093442:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000800:00000010:1.0:1784949191.093443:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000100:00000001:2.0:1784949191.093445:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306130475 00000020:00000010:0.0:1784949191.093460:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:1.0:1784949191.093461:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949191.093464:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306130436 00000800:00000001:3.0:1784949191.093477:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949191.093485:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:0.0:1784949191.093494:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949191.093494:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306130475 00000400:00000200:1.0:1784949191.093496:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000800:00000001:3.0:1784949191.093497:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949191.093506:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000100:00000001:0.0:1784949191.093513:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000800:00000001:3.0:1784949191.093519:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949191.093520:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949191.093523:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949191.093525:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306130436 00000100:00000001:0.0:1784949191.093529:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949191.093532:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949191.093542:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949191.093550:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.093562:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306130475 00000400:00000200:3.0:1784949191.093566:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949191.093576:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949191.093584:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306130436 00000100:00000001:0.0:1784949191.093601:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784949191.093608:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643cb00 00000100:00000001:0.0:1784949191.093609:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949191.093613:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949191.093614:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1784949191.093627:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 797 00000400:00000200:3.0:1784949191.093631:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 174192 00000100:00000040:1.0:1784949191.093634:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949191.093651:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949191.093653:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949191.093657:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949191.093663:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949191.093665:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742038608 : -115538967513008 : ffff96eafce23850) 00000800:00000001:3.0:1784949191.093669:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949191.093683:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949191.093688:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000092af50af x1871651289680512/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:97/0 lens 224/0 e 0 to 0 dl 1784949202 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949191.093694:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949191.093696:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949191.093712:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949191.093736:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949191.093737:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949191.093742:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784949191.093745:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:3.0:1784949191.093748:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784949191.093752:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949191.093753:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784949191.093754:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:3.0:1784949191.093773:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000031b67b62. 00000020:00000010:0.0:1784949191.093775:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:2.0:1784949191.093778:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949191.093784:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000031b67b62 x1871651289680640 msgsize 224 00000100:00000001:2.0:1784949191.093791:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949191.093800:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949191.093803:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000092af50af pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289680512:12345-0@lo:400: 00000100:00100000:3.0:1784949191.093816:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784949191.093821:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949191.093822:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289680512 00000020:00000001:0.0:1784949191.093833:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949191.093843:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949191.093844:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:0.0:1784949191.093848:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949191.093855:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:3.0:1784949191.093857:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949191.093858:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949191.093867:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000001:3.0:1784949191.093869:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949191.093873:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949191.093879:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949191.093882:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949191.093886:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949191.093889:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949191.093893:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949191.093898:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949191.093945:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949191.093950:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289680640 02000000:00000001:1.0:1784949191.093953:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000010:0.0:1784949191.093955:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000efbd7ee7. 00000100:00000001:1.0:1784949191.093956:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949191.093962:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949191.093966:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949191.093969:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949191.093974:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949191.093977:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289680640 00000020:00000001:1.0:1784949191.093986:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:0.0:1784949191.093987:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949191.093989:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949191.093996:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00010000:00000040:0.0:1784949191.093997:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289680512 00000020:00000040:1.0:1784949191.094001:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000001:0.0:1784949191.094007:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:1.0:1784949191.094008:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949191.094016:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949191.094020:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000200:0.0:1784949191.094021:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000092af50af x1871651289680512/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:97/0 lens 224/224 e 0 to 0 dl 1784949202 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949191.094028:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:1.0:1784949191.094036:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949191.094046:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000020:00000010:1.0:1784949191.094067:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004520da55. 00010000:00000001:0.0:1784949191.094069:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949191.094074:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949191.094078:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003a1369bf. 00000100:00001000:0.0:1784949191.094081:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=63 v=5 (1 1 1 1) 00000020:00000010:1.0:1784949191.094086:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949191.094091:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949191.094097:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:1.0:1784949191.094100:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949191.094101:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784949191.094109:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:0.0:1784949191.094110:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784949191.094113:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:0.0:1784949191.094116:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949191.094119:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949191.094123:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949191.094129:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000040:0.0:1784949191.094130:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784949191.094137:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000010:0.0:1784949191.094141:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000001:1.0:1784949191.094141:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949191.094152:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289680512, offset 224 00000400:00000200:0.0:1784949191.094160:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949191.094168:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949191.094177:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949191.094187:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784949191.094193:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ca80 00000100:00000001:1.0:1784949191.094195:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949191.094208:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 773 00000400:00000200:0.0:1784949191.094210:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9189 [1] + 224 00000100:00000040:1.0:1784949191.094226:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000200:0.0:1784949191.094229:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949191.094237:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554959744 : -115544154591872 : ffff96e9c7b5ad80) 00000400:00000200:0.0:1784949191.094244:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949191.094254:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784949191.094259:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000031b67b62 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:97/0 lens 224/0 e 0 to 0 dl 1784949202 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949191.094265:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949191.094311:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949191.094322:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949191.094331:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:1.0:1784949191.094335:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949191.094337:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:0.0:1784949191.094344:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1784949191.094345:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000031b67b62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289680640:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949191.094353:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289680640 00000400:00000200:0.0:1784949191.094356:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000020:00000001:1.0:1784949191.094357:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:0.0:1784949191.094365:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000020:00000001:1.0:1784949191.094366:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949191.094393:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949191.094399:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:1.0:1784949191.094401:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949191.094415:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949191.094417:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949191.094422:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000040:0.0:1784949191.094425:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:1.0:1784949191.094432:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00010000:00000001:0.0:1784949191.094433:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949191.094440:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949191.094441:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949191.094446:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:1.0:1784949191.094449:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949191.094451:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949191.094454:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:0.0:1784949191.094461:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000092af50af x1871651289680512/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:97/0 lens 224/224 e 0 to 0 dl 1784949202 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949191.094464:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.094464:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1784949191.094478:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000001:2.0:1784949191.094484:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784949191.094487:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00100000:0.0:1784949191.094494:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000092af50af pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289680512:12345-0@lo:400: Request processed in 763us (1464us total) trans 0 rc 0/0 00000400:00000001:2.0:1784949191.094494:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1784949191.094495:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949191.094508:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000001f4abcc4. 00000400:00000010:2.0:1784949191.094511:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 02000000:00000001:1.0:1784949191.094516:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949191.094524:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 797 00000100:00000001:1.0:1784949191.094526:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949191.094534:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:1.0:1784949191.094534:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949191.094534:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000100:00000001:0.0:1784949191.094548:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:2.0:1784949191.094551:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00010000:00000040:1.0:1784949191.094552:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289680640 00000100:00000001:0.0:1784949191.094554:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949191.094566:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00010000:00000001:1.0:1784949191.094572:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000010:0.0:1784949191.094578:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949191.094578:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000200:1.0:1784949191.094581:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000031b67b62 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:97/0 lens 224/224 e 0 to 0 dl 1784949202 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949191.094593:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000100:00000200:2.0:1784949191.094603:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949191.094608:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000040:0.0:1784949191.094611:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000001:1.0:1784949191.094617:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949191.094620:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:1.0:1784949191.094633:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=58 v=5 (1 1 1 1) 02000000:00000010:0.0:1784949191.094635:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000efbd7ee7. 02000000:00000001:0.0:1784949191.094648:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949191.094655:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949191.094661:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1784949191.094676:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:1.0:1784949191.094677:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000400:00000010:0.0:1784949191.094691:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ecb3ce7f. 00000100:00000001:1.0:1784949191.094694:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949191.094705:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:0.0:1784949191.094707:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000095fae736. 02000000:00000001:1.0:1784949191.094710:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949191.094711:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949191.094735:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949191.094751:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949191.094769:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000001:2.0:1784949191.094782:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:1.0:1784949191.094783:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289680640, offset 224 00000100:00000001:2.0:1784949191.094794:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949191.094807:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949191.094812:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949191.094820:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000200:1.0:1784949191.094825:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784949191.094829:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784949191.094840:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3721:3721:242:4294967295] 192.168.202.137@tcp LPNI seq info [2388:2388:8:4294967295] 00000100:00000001:2.0:1784949191.094841:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949191.094864:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949191.094880:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00001000:2.0:1784949191.094884:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=63 v=5 (5 5 5 5) 00000800:00000200:1.0:1784949191.094897:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00001000:2.0:1784949191.094908:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=63 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949191.094921:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949191.094925:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949191.094932:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949191.094938:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949191.094946:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c158c1d0. 00000800:00000200:1.0:1784949191.094955:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949191.094964:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949191.094968:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c158c1d0 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949191.094985:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949191.094986:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949191.095001:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949191.095006:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000040:2.0:1784949191.095008:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949191.095011:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949191.095017:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949191.095028:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000031b67b62 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:97/0 lens 224/224 e 0 to 0 dl 1784949202 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949191.095079:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000031b67b62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289680640:12345-192.168.202.137@tcp:400: Request processed in 747us (1282us total) trans 0 rc 0/0 00000100:00000001:2.0:1784949191.095090:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1784949191.095106:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 773 00000100:00000001:2.0:1784949191.095106:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949191.095121:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949191.095122:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949191.095131:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949191.095136:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949191.095141:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003a1369bf. 00000100:00000040:2.0:1784949191.095142:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949191.095149:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949191.095161:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004520da55. 00000020:00000040:1.0:1784949191.095177:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000800:00000200:3.0:1784949191.095190:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949191.095191:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949191.095202:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2b08fd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289680512:0@lo:400: 00000800:00000010:3.0:1784949191.095206:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c158c1d0. 00000400:00000200:3.0:1784949191.095219:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949191.095230:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949191.095236:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949191.095237:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949191.095243:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784949191.095248:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:3.0:1784949191.095254:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000800:00000001:0.0:1784949191.095255:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949191.095257:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2b08fd3 x1871651289680512/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949191.095267:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949191.095268:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949191.095281:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949191.095283:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949191.095315:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949191.095317:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949191.095335:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:0.0:1784949191.095343:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643cb00 02000000:00000010:2.0:1784949191.095343:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001df3b40e. 00000400:00000200:0.0:1784949191.095393:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x918d [1] + 224 02000000:00000001:2.0:1784949191.095397:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000001:0.0:1784949191.095404:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949191.095423:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000800:00000001:0.0:1784949191.095424:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949191.095432:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1784949191.095434:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:0.0:1784949191.095445:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949191.095448:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:0.0:1784949191.095453:0:337114:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:2.0:1784949191.095457:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000635133da. 00000100:00000200:0.0:1784949191.095463:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949191.095470:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949191.095477:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:0.0:1784949191.095484:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784949191.095486:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2b08fd3. 00000100:00000001:2.0:1784949191.095517:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949191.095518:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949191.095528:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949191.095531:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949191.095543:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949191.095546:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949191.095557:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949191.095564:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949191.095573:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949191.095579:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949191.095590:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949191.095599:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949191.095614:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949191.095617:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949191.095621:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c6713e8b. 00000400:00000200:3.0:1784949191.095631:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:3.0:1784949191.095645:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:3.0:1784949191.095651:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949191.095660:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949191.095681:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949191.095721:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949191.095724:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949191.095730:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949191.095734:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949191.095738:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949191.095741:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949191.095745:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949191.095754:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=58 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949191.095758:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=58 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949191.095762:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949191.095764:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949191.095767:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949191.095772:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949191.095782:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949191.095786:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949191.095788:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949191.095790:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949191.095792:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949191.095796:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949191.095804:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949191.095808:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949191.095810:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949191.095814:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949191.095823:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289680640:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949191.095828:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949191.095831:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949191.095834:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289680640/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949191.095849:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949191.095852:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949191.095855:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000378c7327. 02000000:00000001:3.0:1784949191.095860:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949191.095863:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949191.095864:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949191.095867:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949191.095870:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d104522b. 02000000:00000001:3.0:1784949191.095874:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949191.095876:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949191.095878:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000100:00000001:3.0:1784949191.095888:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949191.095890:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949191.095892:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949191.095898:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949191.095899:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949191.095902:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949191.095904:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949191.136037:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306130475, at 4306130476 00000100:00000040:0.0:1784949191.136048:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306130475, at 4306130476 00000100:00000010:0.0:1784949191.136066:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003e9ea21c. 00000100:00000040:2.0:1784949191.136071:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306130475, at 4306130476 00000100:00000010:1.0:1784949191.136072:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00000100:00000040:3.0:1784949191.136113:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306130475, at 4306130476 00000100:00000010:2.0:1784949191.136114:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:3.0:1784949191.136138:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00010000:00000001:2.0:1784949191.924074:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949191.924087:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949191.924098:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949191.924102:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949191.924107:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949191.924111:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949191.924115:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949191.924118:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949191.924121:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949191.924124:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949191.924277:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949191.924290:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949191.924298:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949192.116107:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949192.116124:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949192.116127:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949192.116133:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949192.116137:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949192.116142:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949192.116146:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949192.948019:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949192.948030:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949192.948037:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949192.948038:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949192.948170:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949192.948179:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949192.948185:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949193.140063:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949193.140080:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949193.140083:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949193.140090:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949193.140094:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949193.140099:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949193.140103:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949193.972004:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949193.972015:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949193.972022:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949193.972023:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949193.972090:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949193.972099:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949193.972105:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949194.164028:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949194.164044:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949194.164045:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949194.164050:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949194.164053:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949194.164057:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949194.164060:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949194.996101:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949194.996111:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949194.996119:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949194.996120:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949194.996251:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949194.996261:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949194.996267:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949195.188045:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949195.188062:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949195.188065:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949195.188072:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949195.188076:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949195.188084:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949195.188089:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949196.020082:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949196.020094:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949196.020104:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949196.020107:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949196.020215:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949196.020226:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949196.020233:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949196.212110:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949196.212123:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949196.212134:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00000020:00000040:0.0:1784949196.212144:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949196.212149:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949196.212151:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949196.212165:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949196.212175:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.212177:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949196.212186:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949196.212194:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949196.212196:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00000001:2.0:1784949196.212204:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949196.212210:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949196.212211:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.212225:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949196.212239:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949196.212256:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949196.212259:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949196.212268:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7cbcb5a] to pc [ptlrpcd_00_01+1] req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949196.212320:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.212352:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784949196.212356:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:2.0:1784949196.212357:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949196.212360:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949196.212366:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949196.212374:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949196.212375:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949196.212386:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949196.212392:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949196.212412:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949196.212421:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00000020:00000040:0.0:1784949196.212435:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949196.212443:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949196.212444:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949196.212459:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949196.212459:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949196.212472:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949196.212474:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949196.212482:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1784949196.212484:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000062b4a223. 02000000:00000001:2.0:1784949196.212491:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949196.212493:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949196.212500:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289680768:0@lo:400: 00000100:00000040:0.0:1784949196.212510:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949196.212513:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949196.212526:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949196.212538:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784949196.212538:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949196.212551:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949196.212552:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949196.212566:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000064dc7d78] to pc [ptlrpcd_00_02+2] req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784949196.212567:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001df3b40e. 02000000:00000001:2.0:1784949196.212579:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949196.212596:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000400:00000010:2.0:1784949196.212604:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00000200:2.0:1784949196.212611:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289680768, portal 25 00000100:00000001:2.0:1784949196.212614:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949196.212616:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000100:00000001:0.0:1784949196.212618:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949196.212624:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949196.212626:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11473) 00000100:00000001:2.0:1784949196.212644:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949196.212654:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949196.212657:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000001:1.0:1784949196.212662:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784949196.212662:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289680768, offset 0 00000100:00000001:1.0:1784949196.212675:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949196.212680:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949196.212680:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949196.212690:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949196.212700:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949196.212700:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949196.212714:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949196.212735:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1784949196.212741:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643cb80 00000100:00000001:1.0:1784949196.212742:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949196.212764:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784949196.212766:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x880d [2] + 0 02000000:00000001:1.0:1784949196.212774:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949196.212776:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000037df9850. 02000000:00000001:1.0:1784949196.212786:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949196.212788:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000cd71eba3 00000100:00100000:1.0:1784949196.212797:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289680896:192.168.202.137@tcp:400: 00000400:00000200:2.0:1784949196.212800:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949196.212819:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1784949196.212826:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:1.0:1784949196.212830:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949196.212835:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949196.212835:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cd71eba3 02000000:00000001:1.0:1784949196.212849:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:2.0:1784949196.212849:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000cd71eba3. 02000000:00000010:1.0:1784949196.212865:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 00000100:00000001:2.0:1784949196.212866:0:337120:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:1.0:1784949196.212876:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949196.212877:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949196.212891:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000004a69f61a x1871651289680768 msgsize 224 00000400:00000010:1.0:1784949196.212895:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000768f23df. 00000400:00000010:1.0:1784949196.212941:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00100000:2.0:1784949196.212949:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000200:1.0:1784949196.212951:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289680896, portal 10 00000100:00000040:2.0:1784949196.212953:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949196.212959:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949196.212963:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 00000100:00000001:2.0:1784949196.212965:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784949196.212977:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949196.212978:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000010:2.0:1784949196.212989:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:2.0:1784949196.213006:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949196.213008:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949196.213010:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949196.213014:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289680768 00000100:00000200:2.0:1784949196.213017:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949196.213023:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949196.213032:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:1.0:1784949196.213035:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949196.213042:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949196.213046:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000200:1.0:1784949196.213065:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289680896, offset 0 02000000:00000001:0.0:1784949196.213066:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.213070:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784949196.213080:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784949196.213082:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289680768 00000100:00000040:2.0:1784949196.213085:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949196.213092:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949196.213099:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949196.213108:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784949196.213114:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000040:0.0:1784949196.213118:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949196.213121:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949196.213129:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1784949196.213140:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949196.213140:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784949196.213143:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:0.0:1784949196.213155:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784949196.213160:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949196.213165:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784949196.213169:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:0.0:1784949196.213175:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000400:00000200:1.0:1784949196.213180:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3722:3722:242:4294967295] 192.168.202.137@tcp LPNI seq info [2389:2389:8:4294967295] 00000100:00000001:0.0:1784949196.213184:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949196.213192:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000020:00000010:0.0:1784949196.213204:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000400:00000200:1.0:1784949196.213214:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:0.0:1784949196.213216:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000020:00000010:0.0:1784949196.213227:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949196.213230:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949196.213239:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949196.213245:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306135595 00000800:00000010:1.0:1784949196.213251:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002ddfcc2d. 00000100:00000010:2.0:1784949196.213256:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 00000100:00000040:0.0:1784949196.213259:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000200:1.0:1784949196.213271:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949196.213271:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784949196.213272:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949196.213284:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949196.213289:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.213294:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306135595 00000100:00000001:0.0:1784949196.213303:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949196.213306:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000010:2.0:1784949196.213306:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b9b6859c. 00000800:00000200:1.0:1784949196.213320:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002ddfcc2d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949196.213320:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949196.213342:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306135595 00000100:00000010:2.0:1784949196.213346:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008cd34e64. 00000100:00000001:2.0:1784949196.213350:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949196.213352:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949196.213363:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306135595 00000100:00000010:2.0:1784949196.213372:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f99e6ef2. 00000100:00000001:2.0:1784949196.213380:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949196.213391:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.213393:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949196.213394:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949196.213396:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:1.0:1784949196.213403:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2215 usecs (3 jiffies) 00000100:00000001:2.0:1784949196.213407:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949196.213412:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949196.213413:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949196.213416:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306135556 00000100:00000001:2.0:1784949196.213417:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.213432:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949196.213433:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 798 00000100:00000001:1.0:1784949196.213433:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306135595 00000100:00000001:1.0:1784949196.213448:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306135556 00000100:00000040:0.0:1784949196.213451:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949196.213453:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949196.213453:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949196.213454:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306135595 00000100:00000001:2.0:1784949196.213464:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949196.213467:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742032464 : -115538967519152 : ffff96eafce22050) 00000100:00000001:1.0:1784949196.213478:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306135556 00000100:00000001:3.0:1784949196.213479:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.213484:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949196.213492:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000004a69f61a x1871651289680768/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:102/0 lens 224/0 e 0 to 0 dl 1784949207 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949196.213494:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306135595 00000100:00000001:2.0:1784949196.213494:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949196.213507:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949196.213508:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306135556 00000100:00000001:2.0:1784949196.213527:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949196.213529:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306135595 00000100:00000001:0.0:1784949196.213533:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949196.213546:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949196.213548:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949196.213557:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949196.213558:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949196.213571:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00000001:1.0:1784949196.213572:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949196.213578:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949196.213585:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000004a69f61a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289680768:12345-0@lo:400: 00000100:00000001:1.0:1784949196.213592:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:0.0:1784949196.213593:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289680768 00000100:00000001:1.0:1784949196.213600:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:0.0:1784949196.213601:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949196.213611:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949196.213615:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784949196.213616:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949196.213619:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949196.213621:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784949196.213624:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949196.213628:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949196.213630:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000010:3.0:1784949196.213632:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002ddfcc2d. 00000800:00000001:2.0:1784949196.213634:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949196.213636:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949196.213645:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949196.213650:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949196.213653:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949196.213654:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:3.0:1784949196.213654:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949196.213660:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949196.213662:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949196.213668:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949196.213672:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949196.213680:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:3.0:1784949196.213682:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949196.213688:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:2.0:1784949196.213696:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:0.0:1784949196.213698:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:3.0:1784949196.213699:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 02000000:00000010:0.0:1784949196.213713:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000994a91fc. 00000400:00000200:2.0:1784949196.213715:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643cc00 00000400:00000010:3.0:1784949196.213715:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 02000000:00000001:0.0:1784949196.213730:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949196.213735:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949196.213738:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784949196.213739:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949196.213751:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 174416 00010000:00000040:0.0:1784949196.213753:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289680768 00000100:00000200:3.0:1784949196.213756:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949196.213773:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:2.0:1784949196.213775:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000200:0.0:1784949196.213787:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000004a69f61a x1871651289680768/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:102/0 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949196.213787:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949196.213798:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949196.213801:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784949196.213818:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949196.213829:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949196.213849:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949196.213857:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00010000:00000001:0.0:1784949196.213868:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000010:2.0:1784949196.213870:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f59951e9. 00010000:00000001:0.0:1784949196.213875:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949196.213879:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000f59951e9 x1871651289680896 msgsize 224 00000100:00000001:3.0:1784949196.213881:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949196.213883:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=68 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949196.213893:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1784949196.213900:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949196.213902:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949196.213937:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:2.0:1784949196.213945:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949196.213950:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:2.0:1784949196.213954:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949196.213956:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949196.213963:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949196.213967:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949196.213972:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949196.213981:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949196.213986:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000200:0.0:1784949196.213992:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289680768, offset 224 00000100:00000001:1.0:1784949196.213994:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:0.0:1784949196.214001:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:1.0:1784949196.214002:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289680896 02000000:00000001:1.0:1784949196.214010:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949196.214013:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949196.214016:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949196.214021:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949196.214022:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:1.0:1784949196.214030:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289680896 00000020:00000001:1.0:1784949196.214035:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949196.214038:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:0.0:1784949196.214045:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643cb80 00000020:00000001:1.0:1784949196.214046:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949196.214053:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949196.214059:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784949196.214061:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x91a5 [1] + 224 00000020:00000001:1.0:1784949196.214065:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949196.214072:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949196.214076:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1784949196.214078:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:1.0:1784949196.214085:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949196.214086:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:1.0:1784949196.214095:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:0.0:1784949196.214102:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:1.0:1784949196.214105:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00000100:00000200:0.0:1784949196.214111:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949196.214116:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008c1455b1. 00000020:00000010:1.0:1784949196.214125:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:0.0:1784949196.214139:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949196.214141:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949196.214150:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949196.214154:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949196.214160:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.214169:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949196.214172:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949196.214174:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949196.214179:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949196.214194:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000100:00000001:1.0:1784949196.214197:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949196.214198:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784949196.214210:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949196.214214:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949196.214224:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949196.214229:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949196.214230:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949196.214233:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949196.214236:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000001:2.0:1784949196.214238:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949196.214240:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949196.214244:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:2.0:1784949196.214245:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949196.214247:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949196.214257:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 774 00000400:00000010:2.0:1784949196.214257:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00010000:00000001:0.0:1784949196.214260:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949196.214271:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949196.214272:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000100:00000040:1.0:1784949196.214275:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949196.214277:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949196.214284:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000020:00000001:0.0:1784949196.214286:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949196.214294:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562009600 : -115544147542016 : ffff96e9c8214000) 00000100:00000001:3.0:1784949196.214296:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.214299:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784949196.214306:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004a69f61a x1871651289680768/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:102/0 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949196.214315:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949196.214346:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949196.214349:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f59951e9 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:102/0 lens 224/0 e 0 to 0 dl 1784949207 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949196.214425:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949196.214435:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.214438:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:0.0:1784949196.214444:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000004a69f61a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289680768:12345-0@lo:400: Request processed in 900us (1541us total) trans 0 rc 0/0 00000020:00000010:1.0:1784949196.214447:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:2.0:1784949196.214448:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949196.214460:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949196.214464:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949196.214465:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949196.214474:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949196.214475:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 798 00000020:00000010:1.0:1784949196.214476:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 02000000:00000001:2.0:1784949196.214483:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949196.214492:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00100000:1.0:1784949196.214493:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f59951e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289680896:12345-192.168.202.137@tcp:400: 00000100:00001000:2.0:1784949196.214495:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=68 v=5 (5 5 5 5) 00000100:00000001:0.0:1784949196.214499:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949196.214508:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949196.214520:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00001000:2.0:1784949196.214521:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=68 v=5 (1 1 1 1) 00000100:00000200:1.0:1784949196.214522:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289680896 00000020:00000001:1.0:1784949196.214538:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949196.214539:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:0.0:1784949196.214541:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:2.0:1784949196.214551:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949196.214553:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000010:0.0:1784949196.214554:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00010000:00000001:2.0:1784949196.214561:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:1.0:1784949196.214563:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949196.214566:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949196.214569:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784949196.214576:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949196.214577:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:0.0:1784949196.214580:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:1.0:1784949196.214591:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 02000000:00000010:0.0:1784949196.214592:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000994a91fc. 00000100:00000040:2.0:1784949196.214599:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949196.214607:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:1.0:1784949196.214610:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949196.214615:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949196.214620:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949196.214632:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:1.0:1784949196.214637:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949196.214647:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:0.0:1784949196.214651:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000014751922. 00000100:00000001:2.0:1784949196.214653:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949196.214654:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949196.214667:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a02782b9. 00000100:00000001:2.0:1784949196.214669:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1784949196.214670:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949196.214683:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.214688:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949196.214693:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784949196.214693:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784949196.214708:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006e175381. 00000100:00100000:2.0:1784949196.214710:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e7cbcb5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289680768:0@lo:400: 02000000:00000001:1.0:1784949196.214722:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.214727:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.214728:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.214743:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784949196.214744:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949196.214755:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289680896 00000100:00000040:2.0:1784949196.214757:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e7cbcb5a x1871651289680768/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949196.214766:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949196.214777:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000200:1.0:1784949196.214782:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f59951e9 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:102/0 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949196.214783:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949196.214792:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001df3b40e. 02000000:00000001:2.0:1784949196.214803:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949196.214809:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00010000:00000001:1.0:1784949196.214815:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000040:2.0:1784949196.214815:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949196.214825:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000001:1.0:1784949196.214826:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949196.214828:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 00000100:00001000:1.0:1784949196.214836:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=63 v=5 (1 1 1 1) 02000000:00000001:2.0:1784949196.214839:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949196.214848:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1784949196.214849:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1784949196.214854:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e7cbcb5a. 00000100:00000040:1.0:1784949196.214864:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949196.214868:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949196.214871:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949196.214873:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.214875:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949196.214878:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949196.214879:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949196.214884:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949196.214885:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949196.214896:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000100:00000001:2.0:1784949196.214896:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949196.214901:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949196.214954:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949196.214956:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949196.214966:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289680896, offset 224 00000400:00000200:1.0:1784949196.214976:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949196.214989:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949196.214996:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3723:3723:242:4294967295] 192.168.202.137@tcp LPNI seq info [2390:2390:8:4294967295] 00000400:00000200:1.0:1784949196.215008:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949196.215014:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949196.215018:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a1bdc144. 00000800:00000200:1.0:1784949196.215024:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949196.215030:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949196.215033:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a1bdc144 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949196.215048:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949196.215052:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949196.215056:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949196.215057:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949196.215060:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949196.215065:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f59951e9 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:102/0 lens 224/224 e 0 to 0 dl 1784949207 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949196.215078:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f59951e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289680896:12345-192.168.202.137@tcp:400: Request processed in 640us (1196us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949196.215091:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 774 00000100:00000040:1.0:1784949196.215095:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949196.215098:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949196.215100:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949196.215104:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008c1455b1. 00000020:00000010:1.0:1784949196.215108:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949196.215113:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000020:00000040:1.0:1784949196.215119:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949196.215122:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949196.215226:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949196.215233:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a1bdc144. 00000800:00000001:3.0:1784949196.215237:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949196.215239:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949196.215252:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949196.215254:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949196.215258:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:0.0:1784949196.215263:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000800:00000001:3.0:1784949196.215264:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949196.215268:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949196.215272:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949196.215274:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949196.215287:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949196.215302:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643cc00 00000400:00000200:3.0:1784949196.215311:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x91ad [1] + 224 00000800:00000001:3.0:1784949196.215316:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949196.215343:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949196.215346:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949196.215350:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949196.215357:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949196.215361:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949196.215372:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949196.215392:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949196.215400:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949196.215404:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.215441:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949196.215445:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949196.215452:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949196.215454:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949196.215457:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000768f23df. 00000400:00000200:1.0:1784949196.215462:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:1.0:1784949196.215465:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000100:00000001:1.0:1784949196.215468:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949196.215474:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949196.215486:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949196.215495:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949196.215497:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.215499:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949196.215502:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949196.215505:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949196.215508:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949196.215511:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949196.215518:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=63 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949196.215522:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=63 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949196.215525:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949196.215527:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949196.215530:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949196.215534:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949196.215543:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949196.215546:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949196.215548:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949196.215550:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949196.215552:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949196.215556:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949196.215564:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949196.215567:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949196.215569:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949196.215572:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949196.215581:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000064dc7d78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289680896:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949196.215586:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.215588:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949196.215591:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000064dc7d78 x1871651289680896/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949196.215599:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949196.215602:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949196.215604:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 02000000:00000001:1.0:1784949196.215608:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949196.215610:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949196.215611:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949196.215615:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949196.215617:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000062b4a223. 02000000:00000001:1.0:1784949196.215621:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949196.215623:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949196.215625:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000064dc7d78. 00000100:00000001:1.0:1784949196.215636:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949196.215637:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.215639:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949196.215644:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949196.215645:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949196.215648:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949196.215650:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949196.255971:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306135595, at 4306135596 00000100:00000010:3.0:1784949196.255991:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f99e6ef2. 00000100:00000040:2.0:1784949196.255993:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306135595, at 4306135596 00000100:00000010:2.0:1784949196.256004:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00000100:00000040:1.0:1784949196.256074:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306135595, at 4306135596 00000100:00000040:0.0:1784949196.256088:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306135595, at 4306135596 00000100:00000010:1.0:1784949196.256093:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b9b6859c. 00000100:00000010:0.0:1784949196.256113:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00010000:00000001:2.0:1784949197.044032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949197.044042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949197.044049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949197.044051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949197.044142:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949197.044152:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949197.044157:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949197.236007:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949197.236022:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949197.236024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949197.236029:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949197.236033:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949197.236036:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949197.236039:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949198.068026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949198.068036:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949198.068043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949198.068044:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949198.068127:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949198.068133:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949198.068135:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949198.259999:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949198.260017:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949198.260019:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949198.260026:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949198.260029:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949198.260034:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949198.260037:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949199.092015:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949199.092025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949199.092032:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949199.092034:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949199.092140:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949199.092150:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949199.092158:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949199.283993:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949199.284009:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949199.284011:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949199.284016:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949199.284019:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949199.284023:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949199.284025:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949199.476049:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000800:00000010:1.0:1784949199.476057:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005c3de731. 00000400:00000200:1.0:1784949199.476069:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949199.476076:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949199.476087:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11472 health :900 00000400:00000200:1.0:1784949199.476094:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949199.476099:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:1.0:1784949199.476104:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000800:00000100:1.0:1784949199.476139:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000010:1.0:1784949199.476142:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:1.0:1784949199.476149:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949199.476159:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949199.476167:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949199.476167:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949199.476177:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:1.0:1784949199.476188:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:3.0:1784949199.476192:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000001:3.0:1784949199.476208:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949199.476211:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949199.476215:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 00000400:00000010:3.0:1784949199.476219:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000400:00000200:3.0:1784949199.476228:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949199.476235:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:3.0:1784949199.476242:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006841197d. 00000400:00000200:3.0:1784949199.476247:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000200:3.0:1784949199.476251:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:3.0:1784949199.476254:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000400:00000200:3.0:1784949199.476258:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:3.0:1784949199.476260:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949199.476262:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949199.476263:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:3.0:1784949199.476265:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000400:00000200:3.0:1784949199.476266:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949199.476268:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:3.0:1784949199.476270:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001799e5ef. 00000400:00000200:3.0:1784949199.476277:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:3.0:1784949199.476278:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:3.0:1784949199.476280:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00010000:00000001:2.0:1784949200.116008:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949200.116018:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949200.116025:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949200.116026:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949200.116107:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949200.116112:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949200.116116:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949200.308018:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949200.308035:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949200.308037:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949200.308043:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949200.308046:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949200.308049:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949200.308051:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949200.500027:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000800:00000010:1.0:1784949200.500032:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:1.0:1784949200.500041:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949200.500048:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:1.0:1784949200.500049:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949200.500065:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:3.0:1784949200.500065:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000007028fecb. 00000400:00000200:1.0:1784949200.500076:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000010:3.0:1784949200.500078:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dbce179e. 00000400:00000010:3.0:1784949200.500085:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000400:00000200:1.0:1784949200.500086:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:3.0:1784949200.500111:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e07f0bcd. 00000400:00000200:3.0:1784949200.500117:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949200.500134:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949200.500140:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949200.500144:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3724:3724:245:4294967295] 192.168.202.137@tcp LPNI seq info [2391:2391:8:4294967295] 00000400:00000200:3.0:1784949200.500151:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949200.500157:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949200.500162:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949200.500168:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949200.500174:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949200.500183:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784949200.500184:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949200.500191:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1784949200.500193:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:0.0:1784949200.500204:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949200.500206:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949200.500209:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000400:00000010:0.0:1784949200.500211:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000400:00000200:3.0:1784949200.500214:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11474 health :700 00000400:00000200:0.0:1784949200.500215:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949200.500226:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:0.0:1784949200.500230:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003cdea007. 00000400:00000200:0.0:1784949200.500236:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:0.0:1784949200.500241:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:0.0:1784949200.500245:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000800:00000200:2.0:1784949200.500412:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949200.500417:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000800:00000001:0.0:1784949200.500424:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949200.500425:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949200.500431:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949200.500436:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949200.500438:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784949200.500440:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949200.500444:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949200.500448:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949200.500456:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949200.500470:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949200.500479:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949200.500488:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949200.500493:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3725:3725:245:4294967295] 192.168.202.137@tcp LPNI seq info [2392:2392:8:4294967295] 00000400:00000200:0.0:1784949200.500500:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949200.500503:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949200.500507:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003ed79484. 00000800:00000200:0.0:1784949200.500512:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949200.500517:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949200.500520:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003ed79484 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949200.500595:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949200.500640:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949200.500644:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003ed79484. 00000800:00000001:2.0:1784949200.500645:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949200.500648:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949200.500653:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949200.500655:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949200.500659:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949200.500665:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949200.500666:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949200.500673:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949200.500686:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x91c1 00000800:00000001:2.0:1784949200.500691:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949200.500699:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949200.500701:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949200.500704:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949200.500709:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:2.0:1784949200.500711:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000400:00000200:2.0:1784949200.500713:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949200.500715:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000007028fecb. 00000400:00000010:2.0:1784949200.500717:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dbce179e. 00000800:00000001:2.0:1784949200.500725:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949200.500726:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949201.140000:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949201.140010:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949201.140017:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949201.140021:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949201.140024:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949201.140027:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949201.140030:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949201.140032:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949201.140034:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949201.140036:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949201.140149:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949201.140159:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949201.140164:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949201.332010:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949201.332029:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949201.332031:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949201.332038:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949201.332043:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949201.332048:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.332051:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949201.332083:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949201.332091:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949201.332101:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000020:00000040:3.0:1784949201.332108:0:285579:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949201.332113:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949201.332116:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949201.332123:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949201.332132:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:3.0:1784949201.332140:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949201.332150:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949201.332167:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949201.332169:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949201.332174:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_00_03+3] req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949201.332201:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949201.332203:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949201.332206:0:285579:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949201.332209:0:285579:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949201.332212:0:285579:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949201.332214:0:285579:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000040:3.0:1784949201.332216:0:285579:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949201.332218:0:285579:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949201.332219:0:285579:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949201.332220:0:285579:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949201.332222:0:285579:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949201.332225:0:285579:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949201.332233:0:285579:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949201.332239:0:285579:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949201.332240:0:285579:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949201.332244:0:285579:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_00+0] req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949201.332254:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949201.332266:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949201.332267:0:285579:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949201.332278:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949201.332279:0:285579:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11478) 00000100:00000001:2.0:1784949201.332305:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949201.332311:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949201.332315:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949201.332328:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949201.332331:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949201.332332:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949201.332333:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949201.332346:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949201.332351:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.332369:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949201.332376:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949201.332386:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949201.332393:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1784949201.332394:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949201.332402:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949201.332404:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949201.332415:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289681152:192.168.202.137@tcp:400: 02000000:00000001:0.0:1784949201.332417:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949201.332426:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949201.332434:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949201.332435:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949201.332444:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1784949201.332446:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289681024:0@lo:400: 02000000:00000001:2.0:1784949201.332454:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949201.332457:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949201.332459:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949201.332466:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001df3b40e. 02000000:00000001:0.0:1784949201.332468:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949201.332474:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949201.332475:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949201.332488:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:2.0:1784949201.332493:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 02000000:00000010:0.0:1784949201.332499:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 02000000:00000001:0.0:1784949201.332503:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949201.332505:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000010:0.0:1784949201.332513:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000075be34c1. 00000100:00000200:2.0:1784949201.332514:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289681152, portal 10 00000100:00000001:2.0:1784949201.332522:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949201.332523:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000400:00000010:0.0:1784949201.332526:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000040:2.0:1784949201.332535:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949201.332536:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289681024, portal 25 00000100:00000001:0.0:1784949201.332550:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949201.332558:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554960640 : -115544154590976 : ffff96e9c7b5b100) 00000100:00000001:2.0:1784949201.332573:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949201.332580:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949201.332582:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949201.332591:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000100:00000200:2.0:1784949201.332600:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289681152, offset 0 00000100:00000001:0.0:1784949201.332617:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784949201.332627:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949201.332634:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949201.332641:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000200:0.0:1784949201.332646:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289681024, offset 0 00000400:00000200:2.0:1784949201.332661:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949201.332662:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949201.332676:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949201.332681:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949201.332685:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949201.332688:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949201.332691:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3726:3726:245:4294967295] 192.168.202.137@tcp LPNI seq info [2393:2393:8:4294967295] 00000400:00000200:2.0:1784949201.332702:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949201.332708:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949201.332714:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000005678171. 00000400:00000200:0.0:1784949201.332721:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:2.0:1784949201.332729:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949201.332740:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643cc80 00000800:00000200:2.0:1784949201.332752:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784949201.332759:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000005678171 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1784949201.332764:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8835 [2] + 0 00000400:00000010:0.0:1784949201.332777:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000094ceb1ba. 00000400:00000200:0.0:1784949201.332785:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000042f7d5bc 00000100:00000001:2.0:1784949201.332788:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949201.332794:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2216 usecs (3 jiffies) 00000400:00000200:0.0:1784949201.332795:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949201.332800:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306140675 00000400:00000200:0.0:1784949201.332809:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000010:2.0:1784949201.332815:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008cd34e64. 00000400:00000200:0.0:1784949201.332819:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000042f7d5bc 00000400:00000010:0.0:1784949201.332823:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000042f7d5bc. 00000100:00000001:2.0:1784949201.332827:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784949201.332831:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949201.332835:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:2.0:1784949201.332837:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306140675 00000100:00000040:0.0:1784949201.332842:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000022e78e62 x1871651289681024 msgsize 224 00000100:00000010:2.0:1784949201.332843:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000824e9543. 00000100:00000001:2.0:1784949201.332858:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00100000:0.0:1784949201.332867:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949201.332872:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949201.332874:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306140675 00000100:00000010:2.0:1784949201.332879:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 00000100:00000001:2.0:1784949201.332884:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949201.332888:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306140675 00000100:00000010:2.0:1784949201.332891:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000100:00000001:2.0:1784949201.332894:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949201.332897:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949201.332902:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949201.332951:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.332956:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949201.332958:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000100:00000001:2.0:1784949201.332960:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949201.332966:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000100:00000001:2.0:1784949201.332971:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949201.332975:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949201.332977:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949201.332985:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949201.332986:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949201.332990:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949201.332991:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949201.332997:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949201.332999:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289681024 02000000:00000001:3.0:1784949201.333010:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949201.333011:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949201.333021:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949201.333028:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949201.333048:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949201.333051:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1784949201.333063:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289681024 00000100:00000040:0.0:1784949201.333065:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949201.333070:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949201.333077:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784949201.333088:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949201.333091:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949201.333094:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949201.333096:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784949201.333106:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:3.0:1784949201.333112:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:1.0:1784949201.333115:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949201.333116:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949201.333118:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:2.0:1784949201.333128:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:1.0:1784949201.333132:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000005678171. 00000020:00000010:3.0:1784949201.333132:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:0.0:1784949201.333134:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306140715 00000800:00000001:2.0:1784949201.333147:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949201.333153:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949201.333158:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784949201.333159:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000400:00000200:1.0:1784949201.333162:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949201.333171:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.333172:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306140675 00000020:00000010:3.0:1784949201.333175:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a17860fa. 00000400:00000200:1.0:1784949201.333182:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949201.333198:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000004de9c75. 00000100:00000001:0.0:1784949201.333204:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306140715 00000400:00000200:2.0:1784949201.333204:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949201.333206:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000020:00000010:3.0:1784949201.333208:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000010:1.0:1784949201.333225:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:0.0:1784949201.333233:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306140675 00000100:00000040:3.0:1784949201.333243:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949201.333245:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949201.333245:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643cd00 00000100:00000001:0.0:1784949201.333246:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306140715 00000100:00000001:3.0:1784949201.333250:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949201.333257:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949201.333266:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949201.333268:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949201.333269:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306140675 00000400:00000200:2.0:1784949201.333271:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 174640 00000100:00000001:3.0:1784949201.333279:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949201.333303:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949201.333312:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.333315:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306140715 00000800:00000001:2.0:1784949201.333316:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949201.333329:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949201.333333:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949201.333336:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306140675 00000100:00000001:3.0:1784949201.333344:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949201.333356:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949201.333363:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784949201.333373:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 799 00000100:00000001:2.0:1784949201.333377:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949201.333381:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949201.333382:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000200:2.0:1784949201.333384:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949201.333385:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949201.333391:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742036560 : -115538967515056 : ffff96eafce23050) 00000100:00000040:1.0:1784949201.333395:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784949201.333400:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000002390dcc6. 00000100:00000001:0.0:1784949201.333405:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949201.333408:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000022e78e62 x1871651289681024/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:107/0 lens 224/0 e 0 to 0 dl 1784949212 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949201.333413:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000002390dcc6 x1871651289681152 msgsize 224 00000100:00000001:0.0:1784949201.333425:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.333450:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1784949201.333452:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949201.333461:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949201.333474:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949201.333479:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949201.333487:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:2.0:1784949201.333489:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949201.333498:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949201.333502:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949201.333512:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949201.333512:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949201.333518:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949201.333528:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:1.0:1784949201.333529:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949201.333537:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949201.333540:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784949201.333543:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289681152 00000020:00000010:3.0:1784949201.333546:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 02000000:00000001:2.0:1784949201.333547:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949201.333555:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949201.333558:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949201.333561:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000022e78e62 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289681024:12345-0@lo:400: 02000000:00000001:2.0:1784949201.333562:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949201.333569:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289681152 00000100:00000200:3.0:1784949201.333570:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289681024 00000020:00000001:2.0:1784949201.333574:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:3.0:1784949201.333576:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000040:2.0:1784949201.333577:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949201.333582:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:3.0:1784949201.333585:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000040:2.0:1784949201.333586:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949201.333590:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949201.333590:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949201.333596:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949201.333597:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949201.333601:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784949201.333605:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:3.0:1784949201.333607:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000010:2.0:1784949201.333611:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000020:00000001:3.0:1784949201.333612:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949201.333616:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949201.333617:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949201.333620:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000010:2.0:1784949201.333621:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000020:00000001:3.0:1784949201.333623:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949201.333626:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000010:2.0:1784949201.333628:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003cdea007. 00000100:00000001:3.0:1784949201.333629:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949201.333632:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:2.0:1784949201.333633:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000059fdf11b. 02000000:00000010:3.0:1784949201.333636:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000822137ec. 00000020:00000010:2.0:1784949201.333639:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 02000000:00000001:3.0:1784949201.333640:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949201.333645:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949201.333647:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949201.333649:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289681024 00000100:00000040:2.0:1784949201.333650:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949201.333654:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00010000:00000001:3.0:1784949201.333654:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949201.333657:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00010000:00000200:3.0:1784949201.333661:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000022e78e62 x1871651289681024/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:107/0 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949201.333662:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949201.333669:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00010000:00000001:3.0:1784949201.333671:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949201.333673:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00010000:00000001:3.0:1784949201.333674:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.333675:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949201.333680:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=73 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949201.333685:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949201.333687:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949201.333688:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949201.333689:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949201.333694:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949201.333696:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949201.333699:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949201.333700:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949201.333701:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949201.333705:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:3.0:1784949201.333707:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000100:00100000:2.0:1784949201.333711:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 775 00000100:00000200:3.0:1784949201.333714:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289681024, offset 224 00000100:00000040:2.0:1784949201.333716:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949201.333720:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562014080 : -115544147537536 : ffff96e9c8215180) 00000400:00000200:3.0:1784949201.333723:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:2.0:1784949201.333727:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000002390dcc6 x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:107/0 lens 224/0 e 0 to 0 dl 1784949212 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949201.333735:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949201.333736:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:2.0:1784949201.333740:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:2.0:1784949201.333743:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949201.333745:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000400:00000200:3.0:1784949201.333751:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643cc80 00000100:00100000:2.0:1784949201.333752:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000002390dcc6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289681152:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784949201.333758:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289681152 00000020:00000001:2.0:1784949201.333760:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:3.0:1784949201.333762:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x91c9 [1] + 224 00000020:00000001:2.0:1784949201.333766:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949201.333769:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949201.333771:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949201.333772:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949201.333775:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:3.0:1784949201.333776:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:2.0:1784949201.333781:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949201.333783:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1784949201.333784:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949201.333787:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:3.0:1784949201.333789:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949201.333790:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949201.333793:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949201.333798:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949201.333802:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:3.0:1784949201.333804:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949201.333805:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949201.333810:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002b75a8be. 02000000:00000001:2.0:1784949201.333813:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.333814:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949201.333815:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949201.333818:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289681152 00010000:00000001:2.0:1784949201.333820:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949201.333823:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000002390dcc6 x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:107/0 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949201.333824:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000001:2.0:1784949201.333830:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:3.0:1784949201.333832:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00010000:00000001:2.0:1784949201.333833:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949201.333836:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000100:00001000:2.0:1784949201.333837:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=68 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949201.333840:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949201.333842:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949201.333843:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000040:2.0:1784949201.333847:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949201.333848:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.333851:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000040:3.0:1784949201.333853:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 02000000:00000001:2.0:1784949201.333858:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00010000:00000001:3.0:1784949201.333859:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 02000000:00000001:2.0:1784949201.333863:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949201.333863:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.333868:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1784949201.333868:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949201.333874:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000022e78e62 x1871651289681024/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:107/0 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949201.333879:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949201.333884:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000100:00000200:2.0:1784949201.333889:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289681152, offset 224 00000100:00000001:0.0:1784949201.333894:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949201.333895:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000022e78e62 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289681024:12345-0@lo:400: Request processed in 377us (1044us total) trans 0 rc 0/0 00000400:00000200:2.0:1784949201.333901:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949201.333965:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949201.333967:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949201.333981:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3727:3727:245:4294967295] 192.168.202.137@tcp LPNI seq info [2394:2394:8:4294967295] 00000100:00100000:3.0:1784949201.333986:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 799 00000400:00000001:0.0:1784949201.333988:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949201.334000:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949201.334002:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:3.0:1784949201.334002:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949201.334016:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:0.0:1784949201.334017:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000075be34c1. 00000800:00000200:2.0:1784949201.334019:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949201.334026:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:0.0:1784949201.334036:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000800:00000010:2.0:1784949201.334038:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a21d8ef3. 00000020:00000010:3.0:1784949201.334038:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000004de9c75. 00000400:00000010:0.0:1784949201.334055:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000020:00000010:3.0:1784949201.334058:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000800:00000200:2.0:1784949201.334067:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949201.334069:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949201.334072:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a17860fa. 00000800:00000200:2.0:1784949201.334082:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000200:0.0:1784949201.334083:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949201.334092:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000800:00000200:2.0:1784949201.334097:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a21d8ef3 type 1, nob 320 niov 1 nkiov 1 02000000:00000001:3.0:1784949201.334104:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784949201.334112:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000822137ec. 00000100:00000001:2.0:1784949201.334125:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949201.334128:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949201.334135:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949201.334149:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:2.0:1784949201.334150:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000200:0.0:1784949201.334155:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949201.334157:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c6713e8b. 00010000:00000001:2.0:1784949201.334160:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949201.334168:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949201.334173:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000105270ec. 00000020:00000001:2.0:1784949201.334180:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949201.334196:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002390dcc6 x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:107/0 lens 224/224 e 0 to 0 dl 1784949212 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949201.334201:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949201.334207:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949201.334217:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000002390dcc6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289681152:12345-192.168.202.137@tcp:400: Request processed in 475us (796us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949201.334218:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949201.334229:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949201.334234:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:2.0:1784949201.334236:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 775 00000100:00000001:0.0:1784949201.334242:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949201.334244:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 02000000:00000001:0.0:1784949201.334249:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.334251:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949201.334260:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00001000:0.0:1784949201.334264:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=73 v=5 (5 5 5 5) 00000020:00000010:2.0:1784949201.334265:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000059fdf11b. 00000020:00000010:2.0:1784949201.334295:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949201.334300:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000003cdea007. 00000020:00000040:2.0:1784949201.334303:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949201.334305:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784949201.334310:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=73 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949201.334323:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949201.334327:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949201.334334:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784949201.334339:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949201.334346:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949201.334354:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949201.334358:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:1.0:1784949201.334362:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a21d8ef3. 00000100:00000040:0.0:1784949201.334363:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949201.334369:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949201.334371:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949201.334387:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949201.334399:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949201.334400:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949201.334402:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.334418:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:1.0:1784949201.334423:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000100:00000001:0.0:1784949201.334424:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949201.334432:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:1.0:1784949201.334434:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000100:00000040:0.0:1784949201.334448:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949201.334450:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949201.334457:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949201.334468:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643cd00 00000100:00100000:0.0:1784949201.334493:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289681024:0@lo:400: 00000400:00000200:3.0:1784949201.334495:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x91c5 [1] + 224 00000800:00000001:3.0:1784949201.334522:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949201.334524:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.334532:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949201.334538:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289681024/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949201.334541:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949201.334554:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949201.334572:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949201.334574:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784949201.334578:0:337113:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1784949201.334579:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949201.334589:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 00000100:00000200:3.0:1784949201.334596:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949201.334603:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949201.334614:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000040:3.0:1784949201.334619:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784949201.334621:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949201.334637:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949201.334649:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 00000100:00000001:3.0:1784949201.334657:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1784949201.334658:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949201.334664:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949201.334666:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000800:00000001:3.0:1784949201.334667:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949201.334673:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000800:00000001:3.0:1784949201.334675:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.334677:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.334683:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949201.334689:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949201.334692:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949201.334698:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949201.334703:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.334725:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949201.334730:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949201.334735:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949201.334736:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949201.334738:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000400:00000200:2.0:1784949201.334744:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:2.0:1784949201.334745:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000100:00000001:2.0:1784949201.334748:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949201.334753:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949201.334761:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949201.334766:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949201.334768:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949201.334770:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949201.334772:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949201.334773:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949201.334775:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949201.334777:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949201.334781:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=68 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949201.334783:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=68 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949201.334786:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949201.334787:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949201.334789:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949201.334792:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949201.334797:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.334800:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949201.334801:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949201.334802:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949201.334804:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949201.334806:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949201.334810:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949201.334812:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949201.334814:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949201.334816:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949201.334820:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289681152:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949201.334823:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949201.334825:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949201.334826:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289681152/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949201.334831:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949201.334833:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949201.334834:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001df3b40e. 02000000:00000001:2.0:1784949201.334836:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949201.334837:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949201.334839:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949201.334840:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949201.334842:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:2.0:1784949201.334845:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949201.334846:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949201.334848:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:2.0:1784949201.334850:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949201.334851:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949201.334852:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949201.334856:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949201.334857:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949201.334858:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949201.334860:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949201.376004:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306140715, at 4306140716 00000100:00000040:2.0:1784949201.376018:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306140715, at 4306140716 00000100:00000010:3.0:1784949201.376018:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000100:00000010:2.0:1784949201.376029:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00000100:00000040:1.0:1784949201.376044:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306140715, at 4306140716 00000100:00000010:1.0:1784949201.376060:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000824e9543. 00000100:00000040:0.0:1784949201.376063:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306140715, at 4306140716 00000100:00000010:0.0:1784949201.376078:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00010000:00000001:2.0:1784949202.164016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949202.164025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949202.164033:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949202.164034:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949202.164074:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949202.164079:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949202.164082:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949202.356065:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949202.356083:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949202.356086:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949202.356094:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949202.356098:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949202.356103:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949202.356106:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949203.188065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949203.188076:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949203.188085:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949203.188087:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949203.188189:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949203.188199:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949203.188206:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949203.380044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949203.380065:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949203.380068:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949203.380075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949203.380080:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949203.380085:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949203.380090:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949203.572077:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000800:00000010:1.0:1784949203.572085:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002f91e44d. 00000400:00000200:1.0:1784949203.572098:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949203.572106:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949203.572116:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11476 health :900 00000400:00000200:1.0:1784949203.572124:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949203.572129:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:1.0:1784949203.572134:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:0.0:1784949203.572210:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949203.572222:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:0.0:1784949203.572231:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949203.572234:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949203.572238:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000010:0.0:1784949203.572242:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000400:00000200:0.0:1784949203.572248:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949203.572253:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:0.0:1784949203.572258:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005cc3cd5d. 00000400:00000200:0.0:1784949203.572293:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:0.0:1784949203.572297:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:0.0:1784949203.572312:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00010000:00000001:2.0:1784949204.212036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949204.212045:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949204.212053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949204.212054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949204.212116:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949204.212124:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949204.212129:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949204.404034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949204.404053:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949204.404056:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949204.404062:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949204.404066:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949204.404070:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949204.404073:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949204.595984:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.6@tcp 00000800:00000010:1.0:1784949204.595989:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fbc83221. 00000400:00000200:1.0:1784949204.595998:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949204.596004:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949204.596009:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:1.0:1784949204.596010:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949204.596016:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:1.0:1784949204.596020:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:3.0:1784949204.596020:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000002751ffe5. 00000400:00000010:3.0:1784949204.596046:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f7a7941e. 00000400:00000010:3.0:1784949204.596053:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000400:00000010:3.0:1784949204.596062:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000185e599a. 00000400:00000200:3.0:1784949204.596068:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949204.596071:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949204.596082:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949204.596088:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949204.596093:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3728:3728:247:4294967295] 192.168.202.137@tcp LPNI seq info [2395:2395:8:4294967295] 00000400:00000200:3.0:1784949204.596099:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949204.596106:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949204.596111:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949204.596113:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000800:00000010:3.0:1784949204.596116:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000400:00000001:0.0:1784949204.596118:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949204.596120:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:3.0:1784949204.596122:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949204.596123:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:0.0:1784949204.596125:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:0.0:1784949204.596129:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000200:3.0:1784949204.596131:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:0.0:1784949204.596132:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:0.0:1784949204.596136:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f1904ca. 00000800:00000200:3.0:1784949204.596136:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1784949204.596140:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-110 00000400:00000200:0.0:1784949204.596143:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -110 00000400:00000200:0.0:1784949204.596148:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:3.0:1784949204.596153:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11478 health :800 00000400:00000010:3.0:1784949204.596160:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 00000000029366a5. 00000400:02000040:3.0:1784949204.596165:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000800:00000200:1.0:1784949204.596254:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949204.596278:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:1.0:1784949204.596282:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949204.596286:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784949204.596288:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1784949204.596291:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784949204.596299:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949204.596308:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949204.596310:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949204.596314:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949204.596319:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784949204.596331:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784949204.596338:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784949204.596345:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949204.596348:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3729:3729:247:4294967295] 192.168.202.137@tcp LPNI seq info [2396:2396:8:4294967295] 00000400:00000200:1.0:1784949204.596354:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784949204.596357:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949204.596360:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000af953512. 00000800:00000200:1.0:1784949204.596363:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949204.596366:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949204.596368:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000af953512 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784949204.596403:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949204.596435:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784949204.596443:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949204.596445:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784949204.596446:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000af953512. 00000800:00000001:2.0:1784949204.596455:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949204.596456:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949204.596459:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949204.596466:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949204.596474:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000001:0.0:1784949204.596488:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949204.596497:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x91e1 00000800:00000001:2.0:1784949204.596504:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949204.596519:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949204.596521:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949204.596524:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949204.596529:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000400:00000010:2.0:1784949204.596531:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00000400:00000200:2.0:1784949204.596534:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949204.596537:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000002751ffe5. 00000400:00000010:2.0:1784949204.596540:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f7a7941e. 00000800:00000001:2.0:1784949204.596549:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949204.596550:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949204.597715:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 00000000029366a5. 00010000:00000001:2.0:1784949205.236008:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949205.236015:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949205.236021:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949205.236022:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949205.236129:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949205.236135:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949205.236138:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949205.428001:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949205.428017:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949205.428020:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949205.428025:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949205.428029:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949205.428033:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949205.428037:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949205.620006:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.11@tcp 00000800:00000010:1.0:1784949205.620013:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c2b2d4f6. 00000400:00000200:1.0:1784949205.620024:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949205.620031:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949205.620038:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11478 health :900 00000400:00000200:1.0:1784949205.620044:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949205.620047:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:1.0:1784949205.620052:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:0.0:1784949205.620107:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949205.620118:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:0.0:1784949205.620125:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949205.620127:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949205.620131:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:0.0:1784949205.620133:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000400:00000200:0.0:1784949205.620139:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949205.620142:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:0.0:1784949205.620147:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eb983bef. 00000400:00000200:0.0:1784949205.620152:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:0.0:1784949205.620155:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:0.0:1784949205.620159:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00010000:00000001:2.0:1784949206.260052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949206.260062:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949206.260070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949206.260072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949206.260206:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949206.260214:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949206.260218:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949206.452044:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1784949206.452059:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949206.452062:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949206.452068:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949206.452073:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949206.452076:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949206.452082:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949206.452087:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949206.452088:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006e540bde. 00000100:00000001:0.0:1784949206.452094:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949206.452101:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949206.452109:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949206.452114:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949206.452122:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949206.452136:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000001:3.0:1784949206.452145:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949206.452156:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949206.452171:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949206.452174:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949206.452180:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e540bde] to pc [ptlrpcd_00_01+1] req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949206.452222:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949206.452225:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949206.452230:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949206.452234:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949206.452237:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949206.452272:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003492edb3. 00000020:00000040:3.0:1784949206.452276:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949206.452280:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949206.452282:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949206.452285:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949206.452291:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949206.452296:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949206.452305:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949206.452311:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949206.452318:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949206.452320:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949206.452321:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949206.452324:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949206.452328:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949206.452329:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949206.452343:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949206.452345:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003492edb3] to pc [ptlrpcd_00_02+2] req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949206.452391:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949206.452396:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949206.452402:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949206.452406:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949206.452409:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949206.452414:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289681280:0@lo:400: 00000100:00000001:3.0:1784949206.452418:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949206.452426:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11483) 00000100:00000001:2.0:1784949206.452429:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949206.452439:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949206.452442:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949206.452447:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949206.452455:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000003cdea007. 02000000:00000001:2.0:1784949206.452460:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949206.452469:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784949206.452470:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000100:00100000:0.0:1784949206.452476:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:1.0:1784949206.452479:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784949206.452483:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00000001:0.0:1784949206.452484:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949206.452487:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949206.452498:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:2.0:1784949206.452500:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289681280, portal 25 00000100:00000001:0.0:1784949206.452504:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949206.452507:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949206.452512:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949206.452512:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554957056 : -115544154594560 : ffff96e9c7b5a300) 00000100:00000001:0.0:1784949206.452518:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949206.452523:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949206.452525:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949206.452537:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949206.452546:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949206.452558:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949206.452564:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000100:00000200:2.0:1784949206.452571:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289681280, offset 0 00000100:00000001:0.0:1784949206.452574:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949206.452579:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949206.452584:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:0.0:1784949206.452585:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949206.452592:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949206.452596:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949206.452601:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289681408:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949206.452612:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1784949206.452613:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:0.0:1784949206.452617:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949206.452622:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949206.452625:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949206.452632:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 00000400:00000200:2.0:1784949206.452633:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643cd80 02000000:00000001:0.0:1784949206.452640:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949206.452648:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8859 [2] + 0 00000400:00000010:0.0:1784949206.452649:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000075be34c1. 00000400:00000010:2.0:1784949206.452661:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b62f2953. 00000400:00000200:2.0:1784949206.452671:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b0e9c277 00000400:00000010:0.0:1784949206.452680:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000400:00000200:2.0:1784949206.452683:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1784949206.452691:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289681408, portal 10 00000400:00000200:2.0:1784949206.452693:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949206.452702:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949206.452708:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554958848 : -115544154592768 : ffff96e9c7b5aa00) 00000400:00000200:2.0:1784949206.452708:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0e9c277 00000400:00000010:2.0:1784949206.452717:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b0e9c277. 00000100:00000040:0.0:1784949206.452725:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949206.452728:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949206.452734:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949206.452744:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000db799909 x1871651289681280 msgsize 224 00000100:00000001:0.0:1784949206.452753:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1784949206.452761:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949206.452770:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:0.0:1784949206.452771:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949206.452780:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008441b4a9. 00000100:00000200:0.0:1784949206.452786:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289681408, offset 0 00000400:00000200:0.0:1784949206.452796:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949206.452798:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949206.452816:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949206.452828:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949206.452836:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949206.452842:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949206.452848:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949206.452855:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3730:3730:248:4294967295] 192.168.202.137@tcp LPNI seq info [2397:2397:8:4294967295] 00000400:00000200:0.0:1784949206.452881:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949206.452891:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949206.452898:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003ed79484. 00000400:00000200:2.0:1784949206.452902:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000100:00000001:1.0:1784949206.452940:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949206.452950:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289681280 00000400:00000010:2.0:1784949206.452958:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000800:00000200:0.0:1784949206.452961:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:1.0:1784949206.452961:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949206.452968:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949206.452970:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784949206.452979:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949206.452980:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949206.452983:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 02000000:00000001:1.0:1784949206.452996:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949206.452997:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003ed79484 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:1.0:1784949206.453013:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289681280 00000100:00000001:2.0:1784949206.453014:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784949206.453024:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:2.0:1784949206.453024:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949206.453030:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949206.453032:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949206.453044:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784949206.453045:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2217 usecs (3 jiffies) 00000100:00000001:2.0:1784949206.453048:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949206.453055:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949206.453056:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949206.453058:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306145796 00000020:00000001:1.0:1784949206.453069:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:0.0:1784949206.453073:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:2.0:1784949206.453074:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949206.453075:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949206.453089:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306145835 00000100:00000001:0.0:1784949206.453091:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949206.453092:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949206.453104:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:0.0:1784949206.453109:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306145796 00000100:00000001:1.0:1784949206.453112:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949206.453115:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306145796 00000100:00000010:0.0:1784949206.453119:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000020:00000010:1.0:1784949206.453120:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:2.0:1784949206.453129:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306145835 00000100:00000001:0.0:1784949206.453133:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000010:1.0:1784949206.453139:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004520da55. 00000100:00000001:0.0:1784949206.453143:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306145796 00000100:00000001:2.0:1784949206.453146:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306145796 00000020:00000010:1.0:1784949206.453151:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000e46efe78. 00000100:00000010:0.0:1784949206.453152:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000001:2.0:1784949206.453156:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306145835 00000020:00000010:1.0:1784949206.453162:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949206.453165:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949206.453177:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306145796 00000100:00000010:0.0:1784949206.453183:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:2.0:1784949206.453185:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306145796 00000100:00000040:1.0:1784949206.453186:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949206.453190:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949206.453200:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306145835 00000100:00000001:0.0:1784949206.453203:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949206.453210:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306145796 00000100:00000001:0.0:1784949206.453212:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949206.453214:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949206.453215:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949206.453221:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949206.453226:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949206.453228:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003ed79484. 00000100:00000001:1.0:1784949206.453229:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949206.453230:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949206.453234:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949206.453268:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949206.453270:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949206.453281:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949206.453281:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949206.453291:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949206.453295:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949206.453301:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008441b4a9 00000100:00000001:2.0:1784949206.453308:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:3.0:1784949206.453309:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000008441b4a9. 00000100:00000001:1.0:1784949206.453312:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949206.453314:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949206.453315:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784949206.453321:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949206.453326:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949206.453332:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:3.0:1784949206.453333:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949206.453337:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949206.453339:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949206.453341:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949206.453349:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:0.0:1784949206.453352:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949206.453353:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949206.453362:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949206.453364:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 800 00000100:00000040:1.0:1784949206.453376:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949206.453385:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742029392 : -115538967522224 : ffff96eafce21450) 00000100:00000001:3.0:1784949206.453386:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949206.453387:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:3.0:1784949206.453400:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949206.453403:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000db799909 x1871651289681280/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:112/0 lens 224/0 e 0 to 0 dl 1784949217 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949206.453417:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ce00 00000400:00000200:0.0:1784949206.453445:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 174864 00000100:00000001:3.0:1784949206.453453:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949206.453462:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949206.453464:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784949206.453469:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:1.0:1784949206.453470:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:1.0:1784949206.453478:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949206.453482:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000800:00000001:0.0:1784949206.453483:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949206.453490:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:1.0:1784949206.453496:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000db799909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289681280:12345-0@lo:400: 00000400:00000200:0.0:1784949206.453508:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:1.0:1784949206.453509:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289681280 00000020:00000001:1.0:1784949206.453518:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949206.453522:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:1.0:1784949206.453525:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:0.0:1784949206.453527:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:1.0:1784949206.453532:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949206.453536:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2b08fd3. 00000020:00000001:1.0:1784949206.453538:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949206.453544:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000040:0.0:1784949206.453546:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000b2b08fd3 x1871651289681408 msgsize 224 00000020:00000001:1.0:1784949206.453555:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949206.453561:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949206.453567:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:1.0:1784949206.453570:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949206.453575:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949206.453578:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949206.453581:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949206.453584:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949206.453587:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949206.453590:0:337113:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:1.0:1784949206.453595:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d02c7fef. 02000000:00000001:1.0:1784949206.453600:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949206.453604:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949206.453604:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949206.453611:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949206.453613:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1784949206.453616:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289681280 00010000:00000001:1.0:1784949206.453622:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949206.453628:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000db799909 x1871651289681280/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:112/0 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949206.453641:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949206.453644:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949206.453648:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00001000:1.0:1784949206.453650:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=78 v=5 (1 1 1 1) 00000100:00100000:3.0:1784949206.453658:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289681408 00000100:00000001:1.0:1784949206.453660:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949206.453667:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 02000000:00000001:3.0:1784949206.453669:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949206.453671:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949206.453677:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:1.0:1784949206.453679:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949206.453684:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949206.453686:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949206.453689:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949206.453696:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784949206.453699:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949206.453702:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00100000:3.0:1784949206.453709:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289681408 00000100:00000200:1.0:1784949206.453711:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289681280, offset 224 00000020:00000001:3.0:1784949206.453719:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1784949206.453723:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000040:3.0:1784949206.453726:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949206.453733:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784949206.453740:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000040:3.0:1784949206.453743:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:1.0:1784949206.453753:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643cd80 00000020:00000001:3.0:1784949206.453756:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:1.0:1784949206.453765:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x91e5 [1] + 224 00000020:00000001:3.0:1784949206.453766:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949206.453780:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784949206.453781:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949206.453788:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000400:00000200:1.0:1784949206.453789:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949206.453801:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949206.453803:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949206.453807:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000100:00000200:1.0:1784949206.453812:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949206.453821:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000072ce5ef0. 00000020:00000010:3.0:1784949206.453833:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000004de9c75. 00000100:00000040:1.0:1784949206.453834:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949206.453843:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000040:3.0:1784949206.453860:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949206.453863:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949206.453865:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949206.453868:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949206.453870:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949206.453877:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:1.0:1784949206.453882:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000100:00000001:3.0:1784949206.453883:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000010:1.0:1784949206.453889:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000100:00000001:3.0:1784949206.453891:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949206.453897:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949206.453900:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784949206.453937:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949206.453945:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949206.453950:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949206.453951:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949206.453959:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949206.453967:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784949206.453976:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:1.0:1784949206.453977:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db799909 x1871651289681280/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:112/0 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949206.453981:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949206.453982:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949206.453989:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949206.454000:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784949206.454002:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 776 00000400:00000001:2.0:1784949206.454003:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784949206.454010:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000001:2.0:1784949206.454011:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1784949206.454016:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949206.454020:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949206.454022:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000100:00000001:3.0:1784949206.454023:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554969856 : -115544154581760 : ffff96e9c7b5d500) 00000100:00100000:1.0:1784949206.454032:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000db799909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289681280:12345-0@lo:400: Request processed in 559us (1276us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949206.454034:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949206.454039:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000100:00000040:3.0:1784949206.454044:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b2b08fd3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:112/0 lens 224/0 e 0 to 0 dl 1784949217 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949206.454048:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000100:00000001:2.0:1784949206.454062:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784949206.454069:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 800 00000100:00000001:3.0:1784949206.454071:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949206.454076:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:2.0:1784949206.454077:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949206.454080:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000003ac74adc. 00000100:00000001:1.0:1784949206.454086:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949206.454093:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949206.454094:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949206.454106:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000003ac74adc. 00000020:00000010:1.0:1784949206.454107:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000e46efe78. 00000100:00000200:2.0:1784949206.454115:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949206.454123:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00100000:3.0:1784949206.454133:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b2b08fd3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289681408:12345-192.168.202.137@tcp:400: 00000020:00000010:1.0:1784949206.454136:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004520da55. 00000100:00000001:2.0:1784949206.454138:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949206.454145:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949206.454148:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289681408 00000020:00000040:1.0:1784949206.454151:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784949206.454152:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:3.0:1784949206.454155:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:2.0:1784949206.454160:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:3.0:1784949206.454164:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:2.0:1784949206.454165:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:1.0:1784949206.454166:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:3.0:1784949206.454170:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949206.454173:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d02c7fef. 00000100:00000001:2.0:1784949206.454174:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949206.454176:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949206.454184:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 02000000:00000001:1.0:1784949206.454186:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:2.0:1784949206.454186:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949206.454193:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949206.454194:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949206.454198:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=78 v=5 (5 5 5 5) 00000020:00000001:3.0:1784949206.454200:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949206.454203:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:3.0:1784949206.454212:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00001000:2.0:1784949206.454214:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=78 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949206.454219:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:1.0:1784949206.454220:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000768f23df. 00000100:00000001:2.0:1784949206.454221:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784949206.454227:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949206.454230:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949206.454234:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000b56e9af. 00000020:00000001:3.0:1784949206.454234:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00010000:00000001:2.0:1784949206.454237:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784949206.454271:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00010000:00000001:2.0:1784949206.454273:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949206.454280:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949206.454283:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949206.454289:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007ed348a7. 02000000:00000001:3.0:1784949206.454296:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949206.454298:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949206.454302:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949206.454311:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949206.454319:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289681408 00010000:00000001:3.0:1784949206.454326:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949206.454337:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000200:3.0:1784949206.454337:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b2b08fd3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:112/0 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949206.454349:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949206.454355:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949206.454366:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:2.0:1784949206.454368:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949206.454371:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949206.454381:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=73 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949206.454395:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1784949206.454400:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289681280:0@lo:400: 00000100:00000040:3.0:1784949206.454405:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949206.454414:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784949206.454416:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949206.454419:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784949206.454421:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784949206.454424:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949206.454429:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006e540bde x1871651289681280/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949206.454433:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949206.454446:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949206.454454:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000100:00000001:2.0:1784949206.454456:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000200:3.0:1784949206.454464:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289681408, offset 224 02000000:00000001:2.0:1784949206.454465:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949206.454470:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000003cdea007. 00000400:00000200:3.0:1784949206.454478:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784949206.454480:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949206.454485:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949206.454488:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949206.454495:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:3.0:1784949206.454496:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000010:2.0:1784949206.454499:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 00000400:00000200:3.0:1784949206.454509:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3731:3731:248:4294967295] 192.168.202.137@tcp LPNI seq info [2398:2398:8:4294967295] 02000000:00000001:2.0:1784949206.454510:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949206.454515:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949206.454520:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006e540bde. 00000100:00000001:2.0:1784949206.454526:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949206.454532:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949206.454540:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949206.454545:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784949206.454551:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949206.454558:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949206.454559:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949206.454565:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:3.0:1784949206.454569:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000100:00000001:2.0:1784949206.454571:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949206.454582:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949206.454591:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949206.454595:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949206.454612:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949206.454617:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949206.454620:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949206.454623:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949206.454627:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949206.454632:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2b08fd3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:112/0 lens 224/224 e 0 to 0 dl 1784949217 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949206.454646:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b2b08fd3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289681408:12345-192.168.202.137@tcp:400: Request processed in 574us (1098us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949206.454662:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 776 00000100:00000040:3.0:1784949206.454667:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949206.454671:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949206.454675:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949206.454681:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000004de9c75. 00000020:00000010:3.0:1784949206.454688:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949206.454697:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000072ce5ef0. 00000020:00000040:3.0:1784949206.454704:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949206.454709:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949206.454760:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949206.454765:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:2.0:1784949206.454770:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949206.454777:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949206.454782:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000400:00000010:2.0:1784949206.454784:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000100:00000001:2.0:1784949206.454788:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949206.454790:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949206.454798:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949206.454805:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949206.454808:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949206.454811:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949206.454820:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949206.454835:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ce00 00000400:00000200:2.0:1784949206.454847:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x91ed [1] + 224 00000800:00000001:2.0:1784949206.454858:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949206.454869:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949206.454873:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949206.454880:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949206.454887:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949206.454893:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949206.454945:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949206.454965:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949206.454973:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949206.454978:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949206.455023:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949206.455028:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949206.455034:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949206.455037:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949206.455039:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000075be34c1. 00000400:00000200:0.0:1784949206.455046:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949206.455048:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784949206.455052:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949206.455059:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949206.455071:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949206.455081:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949206.455082:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949206.455085:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949206.455087:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949206.455089:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949206.455091:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949206.455094:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949206.455101:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=73 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949206.455105:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=73 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949206.455108:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949206.455110:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949206.455112:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949206.455116:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949206.455125:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949206.455128:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949206.455130:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949206.455132:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949206.455134:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949206.455137:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949206.455146:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949206.455148:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949206.455150:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949206.455153:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949206.455162:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289681408:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949206.455167:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949206.455169:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949206.455171:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003492edb3 x1871651289681408/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949206.455179:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949206.455181:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949206.455183:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949206.455187:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949206.455189:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949206.455190:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949206.455193:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949206.455195:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:0.0:1784949206.455199:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949206.455201:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949206.455203:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003492edb3. 00000100:00000001:0.0:1784949206.455206:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949206.455207:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949206.455209:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949206.455214:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949206.455216:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949206.455219:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949206.455221:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949206.496004:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306145835, at 4306145836 00000100:00000010:3.0:1784949206.496018:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:0.0:1784949206.496035:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306145835, at 4306145836 00000100:00000040:2.0:1784949206.496040:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306145835, at 4306145836 00000100:00000010:0.0:1784949206.496050:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:2.0:1784949206.496055:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:1.0:1784949206.496061:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306145835, at 4306145836 00000100:00000010:1.0:1784949206.496076:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000800:00000100:1.0:1784949206.644018:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.10@tcp 00000800:00000010:1.0:1784949206.644023:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000400:00000200:1.0:1784949206.644031:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949206.644037:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949206.644042:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11479 health :900 00000400:00000200:1.0:1784949206.644046:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949206.644049:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:1.0:1784949206.644052:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000800:00000100:1.0:1784949206.644071:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000800:00000010:1.0:1784949206.644072:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000400:00000200:0.0:1784949206.644086:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949206.644093:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000200:1.0:1784949206.644096:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000001:0.0:1784949206.644098:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949206.644102:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949206.644103:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949206.644106:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:0.0:1784949206.644109:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000400:00000200:0.0:1784949206.644112:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949206.644115:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949206.644118:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:1.0:1784949206.644121:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000010:0.0:1784949206.644124:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001a4eba99. 00000400:00000200:0.0:1784949206.644132:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-110 00000400:00000200:1.0:1784949206.644134:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:0.0:1784949206.644138:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -110 00000400:00000200:0.0:1784949206.644143:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00000400:00000200:0.0:1784949206.644147:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:0.0:1784949206.644149:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949206.644150:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949206.644151:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:0.0:1784949206.644152:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:0.0:1784949206.644155:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949206.644158:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:0.0:1784949206.644161:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef8e57c8. 00000400:00000200:0.0:1784949206.644165:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:0.0:1784949206.644166:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:0.0:1784949206.644168:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784949207.284027:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949207.284034:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949207.284041:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949207.284042:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949207.284083:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949207.284088:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949207.284092:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949207.475988:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949207.475999:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949207.476001:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949207.476005:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949207.476007:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949207.476010:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949207.476012:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949207.668003:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000400:00000200:3.0:1784949207.668008:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000010:1.0:1784949207.668010:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000010:3.0:1784949207.668018:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000ff2aa94d. 00000400:00000200:1.0:1784949207.668020:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949207.668026:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dd0652fb. 00000400:00000200:1.0:1784949207.668028:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784949207.668033:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000400:00000200:1.0:1784949207.668037:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11480 health :700 00000400:00000010:3.0:1784949207.668042:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001aade4ec. 00000400:00000200:1.0:1784949207.668043:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949207.668051:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:3.0:1784949207.668052:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949207.668057:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:3.0:1784949207.668083:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949207.668090:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949207.668094:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3732:3732:250:4294967295] 192.168.202.137@tcp LPNI seq info [2399:2399:8:4294967295] 00000400:00000200:3.0:1784949207.668100:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949207.668105:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949207.668109:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949207.668113:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000400:00000200:0.0:1784949207.668116:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784949207.668121:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949207.668127:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000800:00000200:3.0:1784949207.668132:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:0.0:1784949207.668133:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:3.0:1784949207.668136:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000001:0.0:1784949207.668137:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949207.668140:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:0.0:1784949207.668143:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000400:00000200:0.0:1784949207.668147:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949207.668150:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:0.0:1784949207.668153:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f11303f8. 00000400:00000200:0.0:1784949207.668157:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:0.0:1784949207.668159:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:0.0:1784949207.668163:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00000800:00000200:1.0:1784949207.668280:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949207.668284:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:1.0:1784949207.668288:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949207.668293:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:3.0:1784949207.668293:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949207.668299:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1784949207.668301:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949207.668304:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949207.668308:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949207.668313:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949207.668320:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949207.668329:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949207.668337:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949207.668343:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949207.668346:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3733:3733:251:4294967295] 192.168.202.137@tcp LPNI seq info [2400:2400:8:4294967295] 00000400:00000200:3.0:1784949207.668352:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949207.668355:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949207.668357:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949207.668360:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949207.668362:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949207.668364:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784949207.668415:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949207.668449:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949207.668452:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:3.0:1784949207.668455:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949207.668456:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949207.668460:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949207.668464:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949207.668483:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949207.668487:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949207.668489:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949207.668497:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784949207.668512:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9201 00000800:00000001:1.0:1784949207.668517:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949207.668525:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949207.668527:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949207.668530:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949207.668534:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:1.0:1784949207.668536:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000400:00000200:1.0:1784949207.668538:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949207.668540:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000ff2aa94d. 00000400:00000010:1.0:1784949207.668543:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dd0652fb. 00000800:00000001:1.0:1784949207.668548:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949207.668550:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949208.308036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949208.308046:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949208.308054:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949208.308055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949208.308181:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949208.308188:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949208.308192:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949208.499977:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949208.499990:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949208.499991:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949208.499996:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949208.499999:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949208.500002:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949208.500004:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949208.692080:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000800:00000010:1.0:1784949208.692088:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f95b4924. 00000400:00000200:1.0:1784949208.692100:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949208.692106:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949208.692114:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11481 health :900 00000400:00000200:1.0:1784949208.692119:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949208.692123:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:1.0:1784949208.692129:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:0.0:1784949208.692188:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949208.692199:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:0.0:1784949208.692207:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949208.692210:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949208.692213:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:0.0:1784949208.692216:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000400:00000200:0.0:1784949208.692246:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949208.692251:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:0.0:1784949208.692256:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006b8bfa56. 00000400:00000200:0.0:1784949208.692267:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000200:0.0:1784949208.692270:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:0.0:1784949208.692274:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00010000:00000001:2.0:1784949209.332044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949209.332054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949209.332062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949209.332064:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949209.332203:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949209.332210:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949209.332214:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949209.523972:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949209.523984:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949209.523985:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949209.523990:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949209.523992:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949209.523995:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949209.523997:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949209.716010:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949209.716021:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000f1e3dc5f. 00000400:00000010:3.0:1784949209.716034:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000072ce5ef0. 00000400:00000010:3.0:1784949209.716043:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 00000800:00000100:1.0:1784949209.716049:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.29@tcp 00000800:00000010:1.0:1784949209.716058:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000010:3.0:1784949209.716058:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b5ecaa25. 00000400:00000200:1.0:1784949209.716075:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949209.716076:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949209.716091:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.29@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949209.716102:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949209.716111:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949209.716116:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3734:3734:252:4294967295] 192.168.202.137@tcp LPNI seq info [2401:2401:8:4294967295] 00000400:00000200:3.0:1784949209.716124:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949209.716132:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949209.716139:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949209.716144:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11482 health :800 00000800:00000010:3.0:1784949209.716147:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000400:00000200:1.0:1784949209.716152:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949209.716161:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.29@tcp: 1 00000800:00000200:3.0:1784949209.716168:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949209.716174:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.29@tcp: -114 00000800:00000200:3.0:1784949209.716185:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949209.716191:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000800:00000100:1.0:1784949209.716252:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000800:00000010:1.0:1784949209.716256:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005523afda. 00000400:00000200:1.0:1784949209.716264:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949209.716270:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949209.716277:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949209.716283:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949209.716284:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:1.0:1784949209.716292:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:0.0:1784949209.716306:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.29@tcp(00000000a5bcc72e) state 0x840 00000400:00000001:0.0:1784949209.716316:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949209.716319:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949209.716323:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:0.0:1784949209.716326:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000400:00000200:0.0:1784949209.716334:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949209.716339:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.29@tcp: -114 00000400:00000010:0.0:1784949209.716345:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000075318cf7. 00000400:00000200:0.0:1784949209.716350:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.29@tcp:-110 00000400:00000200:0.0:1784949209.716353:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.29@tcp(00000000a5bcc72e) state 0x2040 rc -110 00000400:00000200:0.0:1784949209.716357:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.29@tcp 00000400:00000200:0.0:1784949209.716378:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:0.0:1784949209.716381:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949209.716383:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949209.716385:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:0.0:1784949209.716386:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:0.0:1784949209.716389:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949209.716392:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:0.0:1784949209.716396:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000400:00000200:0.0:1784949209.716401:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:0.0:1784949209.716405:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:0.0:1784949209.716409:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000400:00000200:3.0:1784949209.716436:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.29@tcp NID 111.222.173.29@tcp: -110. discovery complete 00000400:00000200:3.0:1784949209.716446:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.173.29@tcp primary NID 111.222.173.29@tcp rc -110 00000800:00000200:2.0:1784949209.716460:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000010:3.0:1784949209.716466:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 000000003ac74adc. 00000800:00000010:2.0:1784949209.716469:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000800:00000001:1.0:1784949209.716476:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949209.716483:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949209.716494:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949209.716500:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000400:00000200:2.0:1784949209.716501:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784949209.716502:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949209.716511:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=000000003ac74adc refcount 1 to 111.222.173.29@tcp 00000800:00000001:1.0:1784949209.716513:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949209.716518:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00010000:00000010:3.0:1784949209.716521:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000029366a5. 00000400:00000200:2.0:1784949209.716531:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:1.0:1784949209.716537:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784949209.716582:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784949209.716607:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784949209.716630:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949209.716647:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3735:3735:254:4294967295] 192.168.202.137@tcp LPNI seq info [2402:2402:8:4294967295] 00000400:00000200:1.0:1784949209.716686:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784949209.716697:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949209.716707:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000af953512. 00010000:00080000:3.0:1784949209.716716:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.173.29@tcp at tail 00000800:00000200:1.0:1784949209.716723:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00010000:00000001:3.0:1784949209.716726:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949209.716734:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949209.716735:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949209.716742:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000af953512 type 3, nob 144 niov 1 nkiov 1 00000100:00000010:3.0:1784949209.716753:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 000000008142e1f1. 00000100:00000400:3.0:1784949209.716761:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.142.29@tcp: rc = -75 00000100:00000040:3.0:1784949209.716765:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.142.29@tcp cnt=32 00000400:00000200:3.0:1784949209.716768:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784949209.716771:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784949209.716775:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784949209.716778:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784949209.716781:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784949209.716787:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784949209.716795:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.716799:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.716804:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784949209.716806:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784949209.716810:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.716832:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784949209.716836:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.716839:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.716846:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.716850:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784949209.716852:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784949209.716859:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000800:00000001:0.0:1784949209.716864:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784949209.716871:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1784949209.716872:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000800:00000001:0.0:1784949209.716877:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949209.716880:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:1.0:1784949209.716881:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000af953512. 00000800:00000001:0.0:1784949209.716887:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949209.716889:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949209.716890:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949209.716899:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949209.716934:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.716939:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000800:00000001:0.0:1784949209.716945:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949209.716950:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:1.0:1784949209.716951:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949209.716955:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949209.716958:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949209.716964:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.716982:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:0.0:1784949209.716983:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949209.716994:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784949209.716999:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949209.717017:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003cdea007. 00000400:00000200:3.0:1784949209.717018:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784949209.717033:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000200:3.0:1784949209.717033:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:0.0:1784949209.717034:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9205 00000400:00000200:3.0:1784949209.717048:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000800:00000001:0.0:1784949209.717051:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949209.717054:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000010:2.0:1784949209.717056:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000023b3e3f1. 00000400:00000200:3.0:1784949209.717064:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000800:00000001:0.0:1784949209.717070:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949209.717074:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949209.717076:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:0.0:1784949209.717079:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949209.717083:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717092:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1784949209.717093:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949209.717101:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717104:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784949209.717106:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784949209.717109:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717122:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:2.0:1784949209.717130:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.717136:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.717142:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.717147:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:2.0:1784949209.717153:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3736:3736:254:4294967295] 111.222.173.1@tcp LPNI seq info [53:53:8:4294967295] 00000400:00000200:2.0:1784949209.717163:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:2.0:1784949209.717174:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.717185:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000400:00000200:0.0:1784949209.717188:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000400:00000200:3.0:1784949209.717191:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000010:0.0:1784949209.717195:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 00000800:00000010:2.0:1784949209.717198:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f495c80e. 00000400:00000200:3.0:1784949209.717198:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1784949209.717204:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000200:3.0:1784949209.717206:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717233:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717236:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000010:0.0:1784949209.717240:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000f1e3dc5f. 00000800:00000200:2.0:1784949209.717243:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949209.717244:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000010:0.0:1784949209.717255:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000072ce5ef0. 00000800:00000200:2.0:1784949209.717256:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000400:00000200:3.0:1784949209.717257:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949209.717275:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000800:00000001:0.0:1784949209.717277:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949209.717282:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000800:00000200:2.1:1784949209.717287:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.1 (cnted 0, interval 60, 11 secs later) 00000400:00000200:3.0:1784949209.717288:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717292:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717300:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717304:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000800:00000001:0.0:1784949209.717307:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:2.0:1784949209.717312:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949209.717314:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000800:00000010:2.0:1784949209.717323:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f495c80e. 00000400:00000200:3.0:1784949209.717327:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717337:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949209.717340:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:3.0:1784949209.717347:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717350:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949209.717356:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949209.717362:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717368:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784949209.717371:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:2.0:1784949209.717372:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949209.717376:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717388:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:3.0:1784949209.717389:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:3.0:1784949209.717395:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949209.717406:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:3.0:1784949209.717414:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717420:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717423:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784949209.717425:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000200:3.0:1784949209.717430:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717435:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:3.0:1784949209.717441:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:3.0:1784949209.717446:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949209.717450:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 rc 0 00000400:00000200:3.0:1784949209.717451:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717461:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717465:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784949209.717467:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:2.0:1784949209.717472:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784949209.717476:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.717488:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000010:2.0:1784949209.717490:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001df3b40e. 00000400:00000200:3.0:1784949209.717491:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717496:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784949209.717502:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000400:00000200:3.0:1784949209.717502:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717512:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000200:3.0:1784949209.717514:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000010:2.0:1784949209.717517:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000043ce20a5. 00000400:00000200:3.0:1784949209.717520:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717533:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949209.717534:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:3.0:1784949209.717543:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717548:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717551:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717553:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784949209.717555:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:2.0:1784949209.717564:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784949209.717565:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717578:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.717585:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.717590:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.717595:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:2.0:1784949209.717600:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3737:3737:254:4294967295] 111.222.142.1@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:2.0:1784949209.717609:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:2.0:1784949209.717621:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.717631:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949209.717634:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784949209.717640:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784949209.717641:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f495c80e. 00000400:00000200:3.0:1784949209.717642:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717651:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717654:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000800:00000200:2.0:1784949209.717660:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949209.717662:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:3.0:1784949209.717672:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949209.717680:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949209.717682:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:3.0:1784949209.717687:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717693:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949209.717698:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:3.0:1784949209.717703:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717709:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:2.0:1784949209.717711:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:3.0:1784949209.717716:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:2.0:1784949209.717727:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784949209.717728:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.717741:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784949209.717742:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717747:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717749:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000010:2.0:1784949209.717750:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006200396a. 00000400:00000200:3.0:1784949209.717752:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:3.0:1784949209.717760:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000010:2.0:1784949209.717761:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000400:00000200:3.0:1784949209.717765:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.717776:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784949209.717777:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949209.717782:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000039173808. 00000400:00000200:3.0:1784949209.717782:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717792:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717796:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:2.0:1784949209.717802:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784949209.717804:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:3.0:1784949209.717815:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717825:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:2.0:1784949209.717832:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.717836:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.717840:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.717845:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:2.0:1784949209.717850:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3738:3738:253:4294967295] 111.222.173.2@tcp LPNI seq info [50:50:8:4294967295] 00000400:00000200:2.0:1784949209.717858:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:2.0:1784949209.717869:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.717877:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:2.0:1784949209.717884:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000400:00000200:3.0:1784949209.717885:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000400:00000200:3.0:1784949209.717891:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.717896:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717936:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000800:00000200:2.0:1784949209.717937:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949209.717939:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:3.0:1784949209.717947:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000800:00000200:2.1:1784949209.717954:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949209.717956:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.717972:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:3.0:1784949209.717975:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784949209.717980:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949209.717983:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:3.0:1784949209.717985:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.717994:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.717997:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784949209.717999:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:2.0:1784949209.718004:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:3.0:1784949209.718006:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.718020:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:2.0:1784949209.718021:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000d531934e. 00000400:00000200:3.0:1784949209.718022:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718027:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784949209.718032:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000200:3.0:1784949209.718033:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718042:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:3.0:1784949209.718045:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000010:2.0:1784949209.718046:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006e91e08a. 00000400:00000200:3.0:1784949209.718051:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.718064:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949209.718064:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000200:3.0:1784949209.718073:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718078:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.718081:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718084:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784949209.718086:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:2.0:1784949209.718093:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:3.0:1784949209.718094:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.718107:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.718113:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.718117:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.718122:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:2.0:1784949209.718127:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3739:3739:252:4294967295] 111.222.142.2@tcp LPNI seq info [50:50:8:4294967295] 00000400:00000200:2.0:1784949209.718135:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:2.0:1784949209.718145:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.718155:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949209.718157:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:3.0:1784949209.718164:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784949209.718166:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000400:00000200:3.0:1784949209.718167:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.718176:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718180:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000800:00000200:2.0:1784949209.718185:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949209.718187:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:3.0:1784949209.718198:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949209.718204:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949209.718206:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:3.0:1784949209.718209:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718211:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.718233:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718236:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:3.0:1784949209.718238:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:3.0:1784949209.718241:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.718246:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784949209.718252:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718257:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.718260:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718262:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:3.0:1784949209.718264:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:3.0:1784949209.718267:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.718271:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784949209.718273:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718276:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949209.718282:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:3.0:1784949209.718283:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718291:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:2.0:1784949209.718296:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x261 rc 0 00000400:00000200:3.0:1784949209.718297:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:3.0:1784949209.718305:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.718317:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784949209.718317:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:3.0:1784949209.718324:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718331:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:2.0:1784949209.718335:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000033c25eed. 00000400:00000200:3.0:1784949209.718336:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718346:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000010:2.0:1784949209.718348:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000400:00000200:3.0:1784949209.718349:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:3.0:1784949209.718361:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949209.718366:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:3.0:1784949209.718368:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949209.718375:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007d16b15b. 00000400:00000200:3.0:1784949209.718375:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.718385:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718390:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000200:2.0:1784949209.718394:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784949209.718396:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000200:3.0:1784949209.718406:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.718416:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:2.0:1784949209.718423:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.718428:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.718433:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.718439:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:2.0:1784949209.718445:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3740:3740:251:4294967295] 111.222.173.3@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:2.0:1784949209.718454:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:2.0:1784949209.718464:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.718475:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:2.0:1784949209.718483:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000094ed938f. 00000400:00000200:3.0:1784949209.718484:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784949209.718491:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718497:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949209.718502:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949209.718507:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000800:00000200:2.0:1784949209.718511:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000400:00000200:3.0:1784949209.718513:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:3.0:1784949209.718520:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:3.0:1784949209.718530:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949209.718534:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949209.718539:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000800:00000200:2.1:1784949209.718545:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.3 (cnted 0, interval 60, 11 secs later) 00000400:00000200:3.0:1784949209.718546:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718550:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.718557:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718561:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000800:00000100:2.0:1784949209.718567:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784949209.718569:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000800:00000010:2.0:1784949209.718576:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000094ed938f. 00000400:00000200:3.0:1784949209.718580:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.718588:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949209.718590:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:3.0:1784949209.718596:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718599:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949209.718609:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949209.718611:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718620:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:3.0:1784949209.718624:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:2.0:1784949209.718625:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949209.718631:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.718638:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:3.0:1784949209.718644:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784949209.718649:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949209.718657:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:3.0:1784949209.718665:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949209.718757:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718761:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784949209.718764:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000200:2.0:1784949209.718775:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784949209.718775:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949209.718789:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 rc 0 00000400:00000200:3.0:1784949209.718792:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000200:3.0:1784949209.718799:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949209.718804:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949209.718810:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:3.0:1784949209.718814:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949209.718823:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000100:00000010:3.0:1784949209.718827:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 000000008142e1f1. 00000400:00000010:2.0:1784949209.718829:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f1904ca. 00000020:00000001:3.0:1784949209.718836:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000010:2.0:1784949209.718840:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00010000:00000001:3.0:1784949209.718842:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000100:00000001:3.0:1784949209.718850:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000010:2.0:1784949209.718852:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000033f377e8. 00000400:00000010:3.0:1784949209.718859:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 0000000043d62367. 00000400:00000200:2.0:1784949209.718867:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784949209.718872:0:343529:0:(peer.c:266:lnet_peer_alloc()) 0000000043d62367 nid 111.222.142.29@tcp 00000400:00000010:3.0:1784949209.718879:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 0000000068c34307. 00000400:00000200:3.0:1784949209.718884:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 0000000068c34307 net tcp 00000400:00000200:2.0:1784949209.718888:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000010:3.0:1784949209.718890:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000a17860fa. 00000400:00000200:2.0:1784949209.718902:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949209.718929:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000a17860fa nid 111.222.142.29@tcp 00000400:00000200:2.0:1784949209.718948:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.718952:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.718956:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:2.0:1784949209.718961:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3741:3741:251:4294967295] 111.222.142.3@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:2.0:1784949209.718968:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:2.0:1784949209.718977:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.718985:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784949209.718992:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.142.29@tcp NID 111.222.142.29@tcp flags 0x0 00000800:00000010:2.0:1784949209.718994:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000094ed938f. 00000400:00000200:3.0:1784949209.719001:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.142.29@tcp: 0 00000800:00000200:2.0:1784949209.719014:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949209.719014:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.29@tcp: 0 00000400:00000200:3.0:1784949209.719021:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.1:1784949209.719031:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:2.0:1784949209.719040:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:2.0:1784949209.719045:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x261 rc 0 00000400:00000200:2.0:1784949209.719055:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000010:2.0:1784949209.719062:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dcf1bf81. 00000400:00000010:2.0:1784949209.719067:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000400:00000010:2.0:1784949209.719075:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000081a06abd. 00000400:00000200:2.0:1784949209.719084:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:2.0:1784949209.719097:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:2.0:1784949209.719103:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.719108:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.719112:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.719118:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:2.0:1784949209.719123:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3742:3742:250:4294967295] 111.222.142.4@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:2.0:1784949209.719130:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:2.0:1784949209.719139:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.719146:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:2.0:1784949209.719152:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c53910d9. 00000800:00000200:2.0:1784949209.719159:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784949209.719166:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:2.0:1784949209.719187:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:2.0:1784949209.719190:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x261 rc 0 00000400:00000200:2.0:1784949209.719196:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:2.0:1784949209.719200:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.719203:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.719206:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:2.0:1784949209.719209:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000400:00000200:2.0:1784949209.719236:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.719244:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:2.0:1784949209.719251:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003cdea007. 00000400:00000200:2.0:1784949209.719258:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-5 00000400:00000200:2.0:1784949209.719265:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.719271:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000400:00000200:2.0:1784949209.719277:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000010:2.0:1784949209.719287:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003cdea007. 00000400:00000010:2.0:1784949209.719293:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000010:2.0:1784949209.719301:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007b996847. 00000400:00000200:2.0:1784949209.719307:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:2.0:1784949209.719318:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:2.0:1784949209.719324:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.719328:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.719332:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.719336:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:2.0:1784949209.719345:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3743:3743:249:4294967295] 111.222.173.5@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:2.0:1784949209.719355:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:2.0:1784949209.719366:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.719375:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000800:00000010:2.0:1784949209.719381:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076916763. 00000800:00000200:2.0:1784949209.719388:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.0:1784949209.719392:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1784949209.719401:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:2.1:1784949209.719404:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.5 (cnted 0, interval 60, 11 secs later) 00000800:00000100:2.0:1784949209.719411:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.5@tcp 00000800:00000010:2.0:1784949209.719414:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000076916763. 00000400:00000200:2.0:1784949209.719420:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949209.719426:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949209.719431:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949209.719435:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:2.0:1784949209.719440:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:2.0:1784949209.719450:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:2.0:1784949209.719453:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 rc 0 00000400:00000200:2.0:1784949209.719460:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:2.0:1784949209.719468:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000072ce2c8. 00000400:00000010:2.0:1784949209.719474:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000010:2.0:1784949209.719480:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001721bb7a. 00000400:00000200:2.0:1784949209.719487:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:2.0:1784949209.719496:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:2.0:1784949209.719501:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.719505:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.719510:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.719513:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:2.0:1784949209.719518:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3744:3744:249:4294967295] 111.222.142.5@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:2.0:1784949209.719526:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:2.0:1784949209.719536:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.719543:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:2.0:1784949209.719549:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076916763. 00000800:00000200:2.0:1784949209.719556:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784949209.719562:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:2.0:1784949209.719569:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:2.0:1784949209.719572:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:2.0:1784949209.719578:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:2.0:1784949209.719584:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000063f28b6f. 00000400:00000010:2.0:1784949209.719588:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000400:00000010:2.0:1784949209.719595:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000339a3bc6. 00000400:00000200:2.0:1784949209.719601:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:2.0:1784949209.719610:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:2.0:1784949209.719614:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.719619:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.719623:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.719628:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:2.0:1784949209.719633:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3745:3745:248:4294967295] 111.222.173.6@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1784949209.719640:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:2.0:1784949209.719651:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.719659:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:2.0:1784949209.719664:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000800:00000200:2.0:1784949209.719671:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784949209.719678:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:2.0:1784949209.719684:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:2.0:1784949209.719688:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:2.0:1784949209.719693:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:2.0:1784949209.719699:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf0e6e18. 00000400:00000010:2.0:1784949209.719703:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000400:00000010:2.0:1784949209.719709:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000059f24104. 00000400:00000200:2.0:1784949209.719715:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:2.0:1784949209.719721:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:2.0:1784949209.719726:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.719730:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.719734:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.719738:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:2.0:1784949209.719743:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3746:3746:247:4294967295] 111.222.142.6@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1784949209.719750:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:2.0:1784949209.719759:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.719767:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:2.0:1784949209.719773:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a21d8ef3. 00000800:00000200:2.0:1784949209.719780:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:2.1:1784949209.719786:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000200:2.0:1784949209.719808:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:2.0:1784949209.719814:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x261 rc 0 00000400:00000200:2.0:1784949209.719820:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:2.0:1784949209.719828:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000010:2.0:1784949209.719832:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000010:2.0:1784949209.719839:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000044d70af9. 00000400:00000200:2.0:1784949209.719845:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:2.0:1784949209.719852:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:2.0:1784949209.719857:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.719861:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.719865:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.719869:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:2.0:1784949209.719874:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3747:3747:246:4294967295] 111.222.173.7@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:2.0:1784949209.719882:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:2.0:1784949209.719894:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.719940:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:2.0:1784949209.719950:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000005678171. 00000800:00000200:2.0:1784949209.719958:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.0:1784949209.719963:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1784949209.719974:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784949209.719980:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.7 (cnted 0, interval 60, 11 secs later) 00000800:00000100:2.0:1784949209.719992:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.7@tcp 00000800:00000010:2.0:1784949209.719996:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000005678171. 00000400:00000200:2.0:1784949209.720004:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949209.720011:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949209.720018:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949209.720024:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:2.0:1784949209.720029:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:2.0:1784949209.720041:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:2.0:1784949209.720047:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 rc 0 00000400:00000200:2.0:1784949209.720054:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:2.0:1784949209.720063:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000684a4a3c. 00000400:00000010:2.0:1784949209.720070:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000400:00000010:2.0:1784949209.720078:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004447c62a. 00000400:00000200:2.0:1784949209.720085:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:2.0:1784949209.720094:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:2.0:1784949209.720099:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.720104:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.720108:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.720113:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:2.0:1784949209.720118:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3748:3748:246:4294967295] 111.222.142.7@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784949209.720126:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:2.0:1784949209.720136:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.720145:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:2.0:1784949209.720150:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000005678171. 00000800:00000200:2.0:1784949209.720158:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784949209.720165:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:2.0:1784949209.720173:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:2.0:1784949209.720177:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:2.0:1784949209.720183:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:2.0:1784949209.720189:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000eae26e4. 00000400:00000010:2.0:1784949209.720195:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000400:00000010:2.0:1784949209.720201:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009ea2352f. 00000400:00000200:2.0:1784949209.720208:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:2.0:1784949209.720244:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:2.0:1784949209.720250:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.720255:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.720259:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.720264:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:2.0:1784949209.720269:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3749:3749:245:4294967295] 111.222.173.8@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:2.0:1784949209.720278:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:2.0:1784949209.720288:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.720297:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784949209.720303:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007a3f8609. 00000800:00000200:2.0:1784949209.720311:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784949209.720319:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:2.0:1784949209.720325:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:2.0:1784949209.720330:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:2.0:1784949209.720336:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:2.0:1784949209.720348:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007db69e48. 00000400:00000010:2.0:1784949209.720353:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000400:00000010:2.0:1784949209.720360:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009b59e4ef. 00000400:00000200:2.0:1784949209.720367:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:2.0:1784949209.720375:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:2.0:1784949209.720380:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.720385:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.720389:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.720393:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:2.0:1784949209.720397:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3750:3750:244:4294967295] 111.222.142.8@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:2.0:1784949209.720405:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:2.0:1784949209.720414:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.720422:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784949209.720428:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cf155122. 00000800:00000200:2.0:1784949209.720435:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784949209.720442:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000400:00000200:2.0:1784949209.720468:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:2.0:1784949209.720472:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x261 rc 0 00000400:00000200:2.0:1784949209.720479:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:2.0:1784949209.720487:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000059e5b358. 00000400:00000010:2.0:1784949209.720494:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000400:00000010:2.0:1784949209.720505:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000051d06b10. 00000400:00000200:2.0:1784949209.720512:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:2.0:1784949209.720524:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:2.0:1784949209.720530:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.720535:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.720540:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.720544:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:2.0:1784949209.720549:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3751:3751:243:4294967295] 111.222.173.9@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:2.0:1784949209.720557:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:2.0:1784949209.720567:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.720578:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:2.0:1784949209.720585:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000800:00000200:2.0:1784949209.720593:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.0:1784949209.720598:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1784949209.720606:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:2.1:1784949209.720611:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.9 (cnted 0, interval 60, 11 secs later) 00000800:00000100:2.0:1784949209.720618:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.9@tcp 00000800:00000010:2.0:1784949209.720622:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000400:00000200:2.0:1784949209.720627:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949209.720634:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949209.720640:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949209.720644:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:2.0:1784949209.720650:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:2.0:1784949209.720659:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:2.0:1784949209.720663:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 rc 0 00000400:00000200:2.0:1784949209.720669:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:2.0:1784949209.720676:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000010:2.0:1784949209.720681:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000010:2.0:1784949209.720688:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000812f4e7a. 00000400:00000200:2.0:1784949209.720693:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:2.0:1784949209.720701:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:2.0:1784949209.720706:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.720710:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.720714:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.720718:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:2.0:1784949209.720723:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3752:3752:243:4294967295] 111.222.173.10@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784949209.720732:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:2.0:1784949209.720742:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.720751:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784949209.720757:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000800:00000200:2.0:1784949209.720766:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784949209.720772:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:2.0:1784949209.720779:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:2.0:1784949209.720784:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:2.0:1784949209.720790:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:2.0:1784949209.720797:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000010:2.0:1784949209.720803:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000492bb2ef. 00000400:00000010:2.0:1784949209.720810:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000040c3d822. 00000400:00000200:2.0:1784949209.720817:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:2.0:1784949209.720825:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:2.0:1784949209.720830:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.720834:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.720837:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.720842:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:2.0:1784949209.720847:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3753:3753:242:4294967295] 111.222.142.10@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784949209.720855:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:2.0:1784949209.720865:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.720873:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:2.0:1784949209.720878:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b9b32818. 00000800:00000200:2.0:1784949209.720885:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:2.1:1784949209.720891:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000400:00000200:2.0:1784949209.720959:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:2.0:1784949209.720963:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x261 rc 0 00000400:00000200:2.0:1784949209.720971:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:2.0:1784949209.720980:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000032fe57fd. 00000400:00000010:2.0:1784949209.720986:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000400:00000010:2.0:1784949209.720994:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005af93f5d. 00000400:00000200:2.0:1784949209.721000:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:2.0:1784949209.721013:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:2.0:1784949209.721019:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.721025:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.721031:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.721036:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:2.0:1784949209.721041:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3754:3754:241:4294967295] 111.222.173.11@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784949209.721049:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:2.0:1784949209.721061:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.721069:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784949209.721077:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ca1124f5. 00000800:00000200:2.0:1784949209.721084:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.0:1784949209.721089:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1784949209.721097:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784949209.721102:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.11 (cnted 0, interval 60, 11 secs later) 00000800:00000100:2.0:1784949209.721110:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784949209.721113:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ca1124f5. 00000400:00000200:2.0:1784949209.721119:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949209.721124:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949209.721129:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949209.721134:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:2.0:1784949209.721138:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:2.0:1784949209.721147:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:2.0:1784949209.721151:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 rc 0 00000400:00000200:2.0:1784949209.721157:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:2.0:1784949209.721166:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009802f36f. 00000400:00000010:2.0:1784949209.721171:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000400:00000010:2.0:1784949209.721178:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000092c6c4bf. 00000400:00000200:2.0:1784949209.721183:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:2.0:1784949209.721192:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:2.0:1784949209.721198:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.721202:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.721207:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.721212:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:2.0:1784949209.721241:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3755:3755:241:4294967295] 111.222.142.11@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:2.0:1784949209.721250:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:2.0:1784949209.721259:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.721268:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:2.0:1784949209.721274:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ca1124f5. 00000800:00000200:2.0:1784949209.721281:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784949209.721288:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:2.0:1784949209.721295:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:2.0:1784949209.721299:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:2.0:1784949209.721306:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:2.0:1784949209.721312:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e6d8b8ad. 00000400:00000010:2.0:1784949209.721318:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000400:00000010:2.0:1784949209.721325:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a2592785. 00000400:00000200:2.0:1784949209.721331:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:2.0:1784949209.721340:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:2.0:1784949209.721345:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.721349:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.721354:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.721358:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:2.0:1784949209.721363:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3756:3756:240:4294967295] 111.222.173.12@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:2.0:1784949209.721370:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:2.0:1784949209.721380:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.721388:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784949209.721394:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cee99234. 00000800:00000200:2.0:1784949209.721401:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784949209.721407:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:2.0:1784949209.721413:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:2.0:1784949209.721416:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:2.0:1784949209.721422:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:2.0:1784949209.721433:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000cdf2ee22. 00000400:00000010:2.0:1784949209.721439:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000400:00000010:2.0:1784949209.721445:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002d887daf. 00000400:00000200:2.0:1784949209.721451:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:2.0:1784949209.721460:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:2.0:1784949209.721464:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.721468:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.721489:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.721493:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:2.0:1784949209.721499:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3757:3757:239:4294967295] 111.222.142.12@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:2.0:1784949209.721507:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:2.0:1784949209.721518:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.721528:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784949209.721535:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000800:00000200:2.0:1784949209.721542:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784949209.721549:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:2.0:1784949209.721579:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:2.0:1784949209.721583:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:2.0:1784949209.721590:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:2.0:1784949209.721598:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 00000400:00000010:2.0:1784949209.721604:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000010:2.0:1784949209.721611:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000db9e65ed. 00000400:00000200:2.0:1784949209.721617:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:2.0:1784949209.721626:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:2.0:1784949209.721631:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.721637:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.721643:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.721664:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:2.0:1784949209.721672:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3758:3758:238:4294967295] 111.222.173.13@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:2.0:1784949209.721682:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:2.0:1784949209.721692:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.721701:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:2.0:1784949209.721708:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000800:00000200:2.0:1784949209.721717:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.0:1784949209.721721:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.13 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1784949209.721730:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784949209.721734:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.13 (cnted 0, interval 60, 11 secs later) 00000800:00000100:2.0:1784949209.721741:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.13@tcp 00000800:00000010:2.0:1784949209.721745:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000400:00000200:2.0:1784949209.721752:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949209.721758:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949209.721763:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949209.721767:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:2.0:1784949209.721772:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:2.0:1784949209.721786:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:2.0:1784949209.721790:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 rc 0 00000400:00000200:2.0:1784949209.721798:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:2.0:1784949209.721806:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000043c3e1d8. 00000400:00000010:2.0:1784949209.721812:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000010:2.0:1784949209.721819:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000096223606. 00000400:00000200:2.0:1784949209.721825:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:2.0:1784949209.721834:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:2.0:1784949209.721838:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.721842:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.721846:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.721851:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:2.0:1784949209.721856:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3759:3759:238:4294967295] 111.222.142.13@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:2.0:1784949209.721864:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:2.0:1784949209.721873:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.721881:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:2.0:1784949209.721886:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000800:00000200:2.0:1784949209.721893:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784949209.721901:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:2.0:1784949209.721966:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:2.0:1784949209.721971:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:2.0:1784949209.721979:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:2.0:1784949209.721989:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004215ddb7. 00000400:00000010:2.0:1784949209.721996:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000400:00000010:2.0:1784949209.722004:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f50feabf. 00000400:00000200:2.0:1784949209.722010:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:2.0:1784949209.722020:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:2.0:1784949209.722025:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.722030:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.722034:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.722038:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:2.0:1784949209.722043:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3760:3760:237:4294967295] 111.222.173.14@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:2.0:1784949209.722051:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:2.0:1784949209.722061:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.722070:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784949209.722075:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000800:00000200:2.0:1784949209.722083:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784949209.722090:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:2.0:1784949209.722095:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:2.0:1784949209.722099:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:2.0:1784949209.722104:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:2.0:1784949209.722111:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005152bcfe. 00000400:00000010:2.0:1784949209.722114:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000010:2.0:1784949209.722121:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000627f8b94. 00000400:00000200:2.0:1784949209.722127:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:2.0:1784949209.722135:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:2.0:1784949209.722140:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.722144:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.722147:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.722151:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:2.0:1784949209.722157:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3761:3761:236:4294967295] 111.222.142.14@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:2.0:1784949209.722164:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:2.0:1784949209.722173:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.722181:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:2.0:1784949209.722187:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c2b2d4f6. 00000800:00000200:2.0:1784949209.722194:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:2.1:1784949209.722201:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:2.0:1784949209.722244:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:2.0:1784949209.722250:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:2.0:1784949209.722258:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:2.0:1784949209.722267:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000098dd29a0. 00000400:00000010:2.0:1784949209.722275:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000010:2.0:1784949209.722286:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000097ad9460. 00000400:00000200:2.0:1784949209.722293:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:2.0:1784949209.722303:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:2.0:1784949209.722309:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.722314:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.722319:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.722324:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:2.0:1784949209.722332:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3762:3762:235:4294967295] 111.222.173.15@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:2.0:1784949209.722342:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:2.0:1784949209.722352:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.722360:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784949209.722366:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000800:00000200:2.0:1784949209.722373:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.0:1784949209.722378:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.15 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1784949209.722386:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784949209.722390:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.15 (cnted 0, interval 60, 11 secs later) 00000800:00000100:2.0:1784949209.722397:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784949209.722400:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:2.0:1784949209.722407:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949209.722413:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949209.722419:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949209.722424:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:2.0:1784949209.722429:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:2.0:1784949209.722438:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:2.0:1784949209.722442:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 rc 0 00000400:00000200:2.0:1784949209.722448:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:2.0:1784949209.722453:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.722456:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.722459:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:2.0:1784949209.722463:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000400:00000200:2.0:1784949209.722468:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.722473:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:2.0:1784949209.722478:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000033c25eed. 00000400:00000200:2.0:1784949209.722485:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-5 00000400:00000200:2.0:1784949209.722490:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.722495:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:2.0:1784949209.722501:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:2.0:1784949209.722506:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000033c25eed. 00000400:00000010:2.0:1784949209.722511:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000400:00000010:2.0:1784949209.722518:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c0f241eb. 00000400:00000200:2.0:1784949209.722539:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:2.0:1784949209.722547:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:2.0:1784949209.722552:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949209.722557:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.722562:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949209.722566:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:2.0:1784949209.722572:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3763:3763:235:4294967295] 111.222.142.15@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:2.0:1784949209.722580:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:2.0:1784949209.722591:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.722601:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:2.0:1784949209.722606:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000800:00000200:2.0:1784949209.722613:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784949209.722621:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:2.0:1784949209.722628:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:2.0:1784949209.722632:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:2.0:1784949209.722638:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.29@tcp(0000000043d62367) state 0x40 00000400:00000010:2.0:1784949209.722662:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c49404b8. 00000400:00000010:2.0:1784949209.722667:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000400:00000010:2.0:1784949209.722673:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000091a5155f. 00000400:00000200:2.0:1784949209.722679:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.29@tcp 00000400:00000200:2.0:1784949209.722686:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.142.29@tcp local destination 00000400:00000200:2.0:1784949209.722692:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949209.722697:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.142.29@tcp 00000400:00000200:2.0:1784949209.722703:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.142.29@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:2.0:1784949209.722710:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3764:3764:234:4294967295] 111.222.142.29@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1784949209.722718:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.29@tcp 00000400:00000200:2.0:1784949209.722728:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.29@tcp(111.222.142.29@tcp:111.222.142.29@tcp) : GET try# 0 00000800:00000200:2.0:1784949209.722736:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.29@tcp 00000800:00000010:2.0:1784949209.722741:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000800:00000010:2.0:1784949209.722751:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000000a5de2ef. 00000800:00000010:2.0:1784949209.722758:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000e6c9da70. 00000800:00000200:2.0:1784949209.722765:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000a5de2ef] -> 12345-111.222.142.29@tcp (2) 00000800:00000200:2.1:1784949209.722772:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000a5de2ef] -> 12345-111.222.142.29@tcp (2) 00000400:00000200:2.0:1784949209.722795:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.29@tcp 00000400:00000200:2.0:1784949209.722799:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.29@tcp(0000000043d62367) state 0x240 rc 0 00000400:00000200:2.0:1784949209.722806:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 00000400:00000001:2.0:1784949209.722811:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.722813:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.722816:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:2.0:1784949209.722819:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000400:00000200:2.0:1784949209.722824:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.722828:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:2.0:1784949209.722833:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003cdea007. 00000400:00000200:2.0:1784949209.722839:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-5 00000400:00000200:2.0:1784949209.722843:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.722849:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:2.0:1784949209.722854:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000001:2.0:1784949209.722858:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.722861:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.722863:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:2.0:1784949209.722867:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:2.0:1784949209.722872:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.722879:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:2.0:1784949209.722885:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089e37819. 00000400:00000200:2.0:1784949209.722895:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-5 00000400:00000200:2.0:1784949209.722900:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.722945:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000400:00000200:2.0:1784949209.722953:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000001:2.0:1784949209.722959:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.722962:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.722965:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000400:00000010:2.0:1784949209.722968:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000400:00000200:2.0:1784949209.722973:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.722979:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:2.0:1784949209.722983:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000059e5b358. 00000400:00000200:2.0:1784949209.722992:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-5 00000400:00000200:2.0:1784949209.722997:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.723002:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000400:00000200:2.0:1784949209.723010:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:2.0:1784949209.723015:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.723018:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.723021:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:2.0:1784949209.723025:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000400:00000200:2.0:1784949209.723030:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.723035:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:2.0:1784949209.723039:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032fe57fd. 00000400:00000200:2.0:1784949209.723045:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-5 00000400:00000200:2.0:1784949209.723050:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.723055:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000400:00000200:2.0:1784949209.723059:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:2.0:1784949209.723064:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.723066:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.723068:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:2.0:1784949209.723071:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000400:00000200:2.0:1784949209.723075:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.723079:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:2.0:1784949209.723083:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 00000400:00000200:2.0:1784949209.723090:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-5 00000400:00000200:2.0:1784949209.723094:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.723099:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000400:00000200:2.0:1784949209.723103:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:2.0:1784949209.723107:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949209.723110:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949209.723112:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:2.0:1784949209.723115:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:2.0:1784949209.723118:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949209.723123:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:2.0:1784949209.723128:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000098dd29a0. 00000400:00000200:2.0:1784949209.723136:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-5 00000400:00000200:2.0:1784949209.723140:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -5 00000400:00000200:2.0:1784949209.723145:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00010000:00000001:2.0:1784949210.356092:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949210.356101:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949210.356109:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949210.356111:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949210.356283:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949210.356292:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949210.356299:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949210.548035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949210.548044:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949210.548061:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949185/real 1784949185] req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949210 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949210.548088:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949210.548093:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949210.548097:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fe908aa6. 00000400:00000200:0.0:1784949210.548110:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:0.0:1784949210.548114:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:0.0:1784949210.548122:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949210.548128:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949210 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949210.548141:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949210 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.548151:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949210.548154:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.548157:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949210.548160:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949210.548165:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949210 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.548183:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.548196:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949210.548198:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.548202:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949210.548257:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949210.548264:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949210.548275:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949210 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.548291:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949210.548297:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.548301:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949210.548304:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.548309:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949210.548314:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949210.548322:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949210.548361:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000040:1.0:1784949210.548367:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949210.548368:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1784949210.548380:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000040:0.0:1784949210.548384:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949210 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949210.548391:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005126bdfa. 00000100:00100000:0.0:1784949210.548401:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289680256:192.168.202.137@tcp:8: 00000020:00000040:1.0:1784949210.548411:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949210.548416:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949210.548421:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949210.548423:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784949210.548427:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1784949210.548433:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000cd114eaa x1871651289680256/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949210 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949210.548436:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.548454:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000010:1.0:1784949210.548455:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000af953512. 02000000:00000001:0.0:1784949210.548460:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1784949210.548467:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949210.548468:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000008618f1b2. 02000000:00000001:0.0:1784949210.548482:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949210.548489:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000040:1.0:1784949210.548489:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000040:0.0:1784949210.548497:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784949210.548508:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949210.548515:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000b91d4ef5. 00000100:00000001:1.0:1784949210.548518:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949210.548523:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949210.548525:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949210.548529:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949210.548532:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000cd114eaa. 00000100:00000040:1.0:1784949210.548533:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005126bdfa] to pc [ptlrpcd_00_03+3] req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.548541:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949210.548548:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.548557:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.548565:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949210.548568:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.548572:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949210.548575:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.548593:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949210.548598:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:2.0:1784949210.548602:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1784949210.548604:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784949210.548612:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949210.548613:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784949210.548621:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784949210.548622:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949210.548632:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:2.0:1784949210.548637:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1784949210.548644:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11458 00000100:00000001:2.0:1784949210.548664:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784949210.548668:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784949210.548670:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949210.548683:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1784949210.548684:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11456 02000000:00000001:2.0:1784949210.548690:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949210.548697:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949210.548702:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289681536:0@lo:400: 00000100:00000001:2.0:1784949210.548707:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:1.0:1784949210.548714:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 02000000:00000001:2.0:1784949210.548715:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949210.548723:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949210.548726:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 02000000:00000001:2.0:1784949210.548729:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784949210.548735:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000010:2.0:1784949210.548739:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000098dd29a0. 00000100:00000001:1.0:1784949210.548741:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949210.548748:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949210.548749:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949210.548757:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:2.0:1784949210.548760:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000056e3aa2. 00000020:00000001:1.0:1784949210.548763:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949210.548769:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000200:0.0:1784949210.548773:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000001:1.0:1784949210.548774:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:2.0:1784949210.548775:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000020:00000040:1.0:1784949210.548783:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000200:2.0:1784949210.548790:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289681536, portal 25 00000400:00000200:0.0:1784949210.548792:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000020:00000001:1.0:1784949210.548793:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:2.0:1784949210.548799:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1784949210.548805:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:2.0:1784949210.548807:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637815424 : -115544071736192 : ffff96e9cca5f480) 00000400:00000010:0.0:1784949210.548811:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000208850ab. 00000100:00000040:1.0:1784949210.548819:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000040:2.0:1784949210.548823:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949210.548824:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000001:1.0:1784949210.548826:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949210.548835:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949210.548844:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:0.0:1784949210.548846:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000225c88a3. 00000020:00000040:1.0:1784949210.548855:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:2.0:1784949210.548863:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949210.548864:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00080000:1.0:1784949210.548870:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000040:2.0:1784949210.548879:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00080000:1.0:1784949210.548885:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000010:2.0:1784949210.548887:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000200:0.0:1784949210.548891:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000200:2.0:1784949210.548899:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289681536, offset 0 00000100:00000001:1.0:1784949210.548901:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949210.548953:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1784949210.548957:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1784949210.548966:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:1.0:1784949210.548968:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949210.548980:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949210.548980:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:1.0:1784949210.548981:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.548993:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:0.0:1784949210.548997:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:1.0:1784949210.549000:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949210.549009:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3765:3765:233:4294967295] 1.2.3.4@tcp LPNI seq info [114:114:8:4294967295] 00000100:00000010:1.0:1784949210.549011:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008223e251. 00000020:00000040:1.0:1784949210.549021:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:0.0:1784949210.549028:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 02000000:00000001:1.0:1784949210.549029:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1784949210.549035:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.549044:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949210.549048:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1784949210.549053:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 02000000:00000001:1.0:1784949210.549056:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949210.549066:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008d04782c. 00000800:00000200:0.0:1784949210.549075:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:1.0:1784949210.549077:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949210.549085:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000010:0.0:1784949210.549092:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008f5595bf. 00000100:00100000:1.0:1784949210.549096:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949210.549115:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ce80 00000800:00000200:0.0:1784949210.549116:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:1.0:1784949210.549117:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949210.549123:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949210.549130:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 6 secs later) 00000400:00000200:2.0:1784949210.549133:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8879 [2] + 0 00000100:00000040:1.0:1784949210.549138:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008223e251] to pc [ptlrpcd_rcv+-1] req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949210.549148:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a86c77e2. 00000800:00000200:0.1:1784949210.549157:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:2.0:1784949210.549161:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000995a12aa 00000800:00000200:0.1:1784949210.549170:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 6 secs later) 00000400:00000200:2.0:1784949210.549179:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949210.549186:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1784949210.549188:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000100:00000040:1.0:1784949210.549195:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1784949210.549198:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008f5595bf. 00000100:00000001:1.0:1784949210.549227:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:0.0:1784949210.549231:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949210.549237:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000010:1.0:1784949210.549238:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006a448376. 00000020:00000040:1.0:1784949210.549245:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:0.0:1784949210.549258:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000001:1.0:1784949210.549258:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1784949210.549259:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000995a12aa 02000000:00000001:1.0:1784949210.549265:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1784949210.549269:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000995a12aa. 02000000:00000001:1.0:1784949210.549272:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949210.549280:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000009dcd7bb6. 00000100:00000001:2.0:1784949210.549282:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949210.549285:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949210.549287:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1784949210.549290:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000040:1.0:1784949210.549298:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784949210.549298:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000001a09b990 x1871651289681536 msgsize 224 00000400:00000200:0.0:1784949210.549308:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00100000:2.0:1784949210.549325:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:0.0:1784949210.549329:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000040:2.0:1784949210.549337:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949210.549348:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949210.549351:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949210.549356:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949210.549357:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:1.0:1784949210.549364:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a448376] to pc [ptlrpcd_00_00+0] req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949210.549372:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000400:00000200:2.0:1784949210.549375:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000100:00000001:3.0:1784949210.549383:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:2.0:1784949210.549384:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000400:00000200:0.0:1784949210.549393:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000100:00100000:3.0:1784949210.549394:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289681536 00000100:00000001:2.0:1784949210.549397:0:337122:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784949210.549404:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000001:0.0:1784949210.549409:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949210.549410:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784949210.549412:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949210.549417:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949210.549418:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949210.549425:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000100:00000001:1.0:1784949210.549426:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949210.549432:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949210.549436:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000040:1.0:1784949210.549440:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11487) 00000100:00100000:3.0:1784949210.549445:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289681536 00000400:00000200:0.0:1784949210.549449:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000020:00000001:3.0:1784949210.549454:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949210.549459:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:2.0:1784949210.549461:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949210.549465:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949210.549467:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000020:00000001:3.0:1784949210.549468:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949210.549472:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949210.549473:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949210.549477:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000010:0.0:1784949210.549480:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000208850ab. 00000020:00000001:3.0:1784949210.549490:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784949210.549498:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:1.0:1784949210.549499:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949210.549501:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949210.549508:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949210.549514:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:1.0:1784949210.549518:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949210.549520:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949210.549520:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949210.549527:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:2.0:1784949210.549530:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784949210.549533:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000040:1.0:1784949210.549536:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949210.549541:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949210.549544:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949210.549554:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306149931 00000020:00000010:3.0:1784949210.549557:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000100:00000001:0.0:1784949210.549560:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:2.0:1784949210.549570:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 00000100:00000001:0.0:1784949210.549572:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949210.549574:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000010:3.0:1784949210.549578:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dbce179e. 00000100:00000001:0.0:1784949210.549580:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949210.549582:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949210.549586:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1784949210.549591:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000010:3.0:1784949210.549591:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000d656743e. 00000100:00000040:0.0:1784949210.549596:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949210.549599:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306149931 02000000:00000001:1.0:1784949210.549600:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949210.549602:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 02000000:00000001:1.0:1784949210.549616:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949210.549617:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000abbd7d53. 00000100:00000001:0.0:1784949210.549624:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1784949210.549625:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949210.549630:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949210.549632:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949210.549634:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289681664:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784949210.549635:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949210.549641:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306149931 00000100:00000001:3.0:1784949210.549642:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:0.0:1784949210.549645:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949210.549651:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949210.549653:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949210.549654:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1784949210.549656:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000082553b41. 00000100:00000001:1.0:1784949210.549663:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949210.549664:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949210.549665:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949210.549667:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949210.549671:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.549682:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949210.549683:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306149931 00000100:00100000:0.0:1784949210.549686:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006a448376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289681792:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949210.549688:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784949210.549693:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f68fd7be. 00000100:00000001:3.0:1784949210.549694:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949210.549700:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949210.549707:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949210.549709:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949210.549710:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949210.549715:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949210.549721:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949210.549722:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949210.549722:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949210.549724:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:0.0:1784949210.549732:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.549733:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949210.549734:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949210.549735:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.549741:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949210.549747:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949210.549747:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949210.549750:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949210.549752:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 801 02000000:00000010:0.0:1784949210.549759:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 00000100:00000040:3.0:1784949210.549763:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949210.549765:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1784949210.549766:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949210.549774:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534742034512 : -115538967517104 : ffff96eafce22850) 00000100:00000001:2.0:1784949210.549775:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000010:0.0:1784949210.549778:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000075be34c1. 00000100:00000001:2.0:1784949210.549789:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784949210.549791:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000001:1.0:1784949210.549793:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949210.549794:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001a09b990 x1871651289681536/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:116/0 lens 224/0 e 0 to 0 dl 1784949221 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949210.549796:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949210.549804:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949210.549806:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289681792, portal 10 00000100:00000001:2.0:1784949210.549807:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.549820:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949210.549821:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949210.549826:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949210.549828:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637802880 : -115544071748736 : ffff96e9cca5c380) 00000020:00000010:3.0:1784949210.549834:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000ffa65c47. 00000100:00000001:3.0:1784949210.549841:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949210.549845:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949210.549848:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000ffa65c47. 00000100:00100000:3.0:1784949210.549865:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001a09b990 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289681536:12345-0@lo:400: 00000100:00000001:0.0:1784949210.549866:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949210.549879:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:3.0:1784949210.549882:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289681536 00000400:00000010:0.0:1784949210.549886:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002c629ffc. 00000020:00000001:3.0:1784949210.549888:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000200:0.0:1784949210.549895:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289681792, offset 0 00000020:00000001:3.0:1784949210.549897:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949210.549902:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949210.549947:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784949210.549959:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:0.0:1784949210.549961:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:3.0:1784949210.549966:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:0.0:1784949210.549971:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784949210.549974:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:0.0:1784949210.549981:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000001:3.0:1784949210.549982:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949210.549990:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:0.0:1784949210.549991:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1784949210.549994:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:0.0:1784949210.549999:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:3.0:1784949210.550001:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949210.550010:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3766:3766:233:4294967295] 192.168.202.137@tcp LPNI seq info [2403:2403:8:4294967295] 00000020:00000001:3.0:1784949210.550011:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949210.550017:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949210.550023:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949210.550031:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000579609a4. 00000400:00000200:0.0:1784949210.550032:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:3.0:1784949210.550040:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949210.550047:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949210.550048:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949210.550055:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949210.550057:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008f5595bf. 00010000:00000040:3.0:1784949210.550060:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289681536 00010000:00000001:3.0:1784949210.550067:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000200:0.0:1784949210.550071:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00010000:00000200:3.0:1784949210.550077:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001a09b990 x1871651289681536/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:116/0 lens 224/224 e 0 to 0 dl 1784949221 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784949210.550092:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00010000:00000001:3.0:1784949210.550102:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000800:00000200:0.0:1784949210.550103:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f5595bf type 1, nob 320 niov 1 nkiov 1 00010000:00000001:3.0:1784949210.550108:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949210.550114:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=82 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949210.550120:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949210.550123:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949210.550125:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949210.550130:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784949210.550131:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949210.550134:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949210.550140:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2218 usecs (3 jiffies) 00000100:00000001:3.0:1784949210.550142:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949210.550149:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306149893 00000100:00000040:3.0:1784949210.550153:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949210.550158:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306149931 00000400:00000010:3.0:1784949210.550162:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000001:0.0:1784949210.550163:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306149893 00000100:00000001:0.0:1784949210.550173:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306149931 00000100:00000200:3.0:1784949210.550175:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289681536, offset 224 00000100:00000001:0.0:1784949210.550179:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306149893 00000100:00000001:0.0:1784949210.550186:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306149931 00000100:00000001:0.0:1784949210.550194:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306149893 00000400:00000200:3.0:1784949210.550194:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949210.550232:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306149931 00000100:00000001:0.0:1784949210.550240:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949210.550244:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949210.550250:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.550259:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.550264:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949210.550266:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ce80 00000100:00000001:0.0:1784949210.550278:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:1.0:1784949210.550286:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949210.550288:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:3.0:1784949210.550289:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x927d [1] + 224 00000800:00000010:1.0:1784949210.550295:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008f5595bf. 00000100:00000001:0.0:1784949210.550304:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:2.0:1784949210.550306:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949210.550308:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949210.550311:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949210.550313:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949210.550320:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949210.550324:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949210.550327:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949210.550332:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949210.550332:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:2.0:1784949210.550336:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949210.550347:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002c629ffc 00000800:00000001:2.0:1784949210.550352:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949210.550353:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002c629ffc. 00000100:00000001:1.0:1784949210.550365:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949210.550370:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949210.550380:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949210.550382:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:3.0:1784949210.550383:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949210.550414:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643cf80 00000400:00000200:2.0:1784949210.550438:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 175088 00000100:00000001:1.0:1784949210.550442:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949210.550454:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949210.550455:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949210.550456:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949210.550466:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949210.550477:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949210.550496:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949210.550501:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.550512:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949210.550514:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949210.550521:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:2.0:1784949210.550528:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000067c965f9. 00000100:00000040:2.0:1784949210.550532:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000067c965f9 x1871651289681792 msgsize 224 00000100:00000001:3.0:1784949210.550539:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:2.0:1784949210.550540:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784949210.550554:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949210.550555:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:3.0:1784949210.550561:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00000001:3.0:1784949210.550566:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949210.550570:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949210.550578:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949210.550584:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949210.550588:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949210.550592:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784949210.550595:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00100000:1.0:1784949210.550602:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289681792 00010000:00000001:3.0:1784949210.550603:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949210.550605:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949210.550610:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949210.550611:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949210.550613:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949210.550618:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949210.550619:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000001:0.0:1784949210.550624:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784949210.550626:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784949210.550630:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1784949210.550631:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001a09b990 x1871651289681536/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:116/0 lens 224/224 e 0 to 0 dl 1784949221 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949210.550634:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949210.550638:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000056e3aa2. 00000100:00100000:1.0:1784949210.550647:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289681792 00000400:00000200:0.0:1784949210.550651:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000020:00000001:1.0:1784949210.550656:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:0.0:1784949210.550659:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000020:00000040:1.0:1784949210.550663:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784949210.550671:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1784949210.550673:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949210.550680:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00100000:3.0:1784949210.550680:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001a09b990 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289681536:12345-0@lo:400: Request processed in 849us (1376us total) trans 0 rc 0/0 00000100:00000200:0.0:1784949210.550682:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949210.550692:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949210.550701:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949210.550713:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:3.0:1784949210.550717:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 801 00000020:00000010:1.0:1784949210.550722:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000040:3.0:1784949210.550729:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949210.550734:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784949210.550737:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949210.550737:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000010:1.0:1784949210.550744:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:3.0:1784949210.550746:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949210.550756:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000d656743e. 00000020:00000010:1.0:1784949210.550759:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004520da55. 00000100:00000001:0.0:1784949210.550762:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784949210.550767:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949210.550769:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949210.550770:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000056a1a23f. 00000100:00000001:0.0:1784949210.550777:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:3.0:1784949210.550778:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dbce179e. 00000020:00000010:1.0:1784949210.550781:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 02000000:00000001:0.0:1784949210.550784:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:3.0:1784949210.550790:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784949210.550791:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:1.0:1784949210.550797:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949210.550798:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949210.550801:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949210.550809:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000010:3.0:1784949210.550810:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000579609a4. 02000000:00000001:0.0:1784949210.550811:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.550814:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784949210.550822:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949210.550823:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784949210.550825:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=82 v=5 (5 5 5 5) 00000100:00000001:3.0:1784949210.550829:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949210.550836:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00001000:0.0:1784949210.550844:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=82 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949210.550845:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:3.0:1784949210.550845:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949210.550852:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.550860:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:3.0:1784949210.550860:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fe908aa6. 00000100:00000001:0.0:1784949210.550865:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949210.550871:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:3.0:1784949210.550872:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007de46116. 00000100:00000001:1.0:1784949210.550874:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784949210.550878:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.550885:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949210.550893:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949210.550897:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949210.550949:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949210.550953:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949210.550956:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949210.550957:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949210.550966:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949210.550971:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949210.550976:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 777 00000100:00000001:2.0:1784949210.550976:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:1.0:1784949210.550988:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949210.550992:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.550997:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562021248 : -115544147530368 : ffff96e9c8216d80) 00000100:00100000:0.0:1784949210.550998:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289681536:0@lo:400: 00000100:00000001:2.0:1784949210.551000:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949210.551015:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000067c965f9 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:116/0 lens 224/0 e 0 to 0 dl 1784949221 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.551016:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551025:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949210.551035:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005126bdfa x1871651289681536/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949210.551043:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949210.551050:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:0.0:1784949210.551054:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1784949210.551056:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949210.551060:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:1.0:1784949210.551062:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 02000000:00000010:0.0:1784949210.551066:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000098dd29a0. 00000100:00100000:1.0:1784949210.551074:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000067c965f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289681792:12345-192.168.202.137@tcp:400: 02000000:00000001:0.0:1784949210.551076:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949210.551082:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000200:1.0:1784949210.551085:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289681792 00000020:00000040:0.0:1784949210.551087:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949210.551092:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949210.551095:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:1.0:1784949210.551099:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000010:0.0:1784949210.551100:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000af953512. 00000020:00000001:1.0:1784949210.551106:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949210.551108:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:1.0:1784949210.551114:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:0.0:1784949210.551115:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:1.0:1784949210.551121:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000010:0.0:1784949210.551122:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005126bdfa. 00000100:00000001:0.0:1784949210.551131:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1784949210.551133:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949210.551136:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949210.551139:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551141:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949210.551146:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784949210.551151:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1784949210.551152:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784949210.551159:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949210.551160:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949210.551170:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949210.551171:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949210.551176:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949210.551177:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949210.551183:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949210.551189:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d8bbc1a8. 02000000:00000001:1.0:1784949210.551192:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.551195:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949210.551198:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949210.551201:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289681792 00010000:00000001:1.0:1784949210.551205:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949210.551233:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000067c965f9 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:116/0 lens 224/224 e 0 to 0 dl 1784949221 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949210.551244:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949210.551246:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949210.551249:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=77 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949210.551254:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949210.551256:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949210.551260:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949210.551263:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949210.551265:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949210.551268:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949210.551271:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949210.551276:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00000200:1.0:1784949210.551280:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289681792, offset 224 00000400:00000200:1.0:1784949210.551286:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949210.551296:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949210.551302:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3767:3767:233:4294967295] 192.168.202.137@tcp LPNI seq info [2404:2404:8:4294967295] 00000400:00000200:1.0:1784949210.551313:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949210.551319:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949210.551323:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000af953512. 00000800:00000200:1.0:1784949210.551329:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949210.551335:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949210.551340:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000af953512 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949210.551356:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949210.551360:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949210.551363:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949210.551365:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949210.551367:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949210.551371:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000067c965f9 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:116/0 lens 224/224 e 0 to 0 dl 1784949221 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949210.551383:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000067c965f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289681792:12345-192.168.202.137@tcp:400: Request processed in 340us (849us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949210.551393:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 777 00000100:00000040:1.0:1784949210.551397:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949210.551401:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949210.551403:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949210.551408:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000056a1a23f. 00000020:00000010:1.0:1784949210.551413:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949210.551417:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004520da55. 00000020:00000040:1.0:1784949210.551422:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949210.551425:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949210.551529:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949210.551535:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000af953512. 00000400:00000200:0.0:1784949210.551542:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949210.551550:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949210.551555:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949210.551556:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:0.0:1784949210.551562:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000100:00000001:0.0:1784949210.551567:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784949210.551569:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551571:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949210.551579:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949210.551586:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949210.551594:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949210.551607:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643cf80 00000400:00000200:1.0:1784949210.551615:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9289 [1] + 224 00000800:00000001:1.0:1784949210.551621:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949210.551631:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949210.551634:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949210.551638:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949210.551644:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949210.551647:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949210.551658:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949210.551680:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949210.551687:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949210.551690:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551732:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949210.551737:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949210.551742:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949210.551743:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949210.551746:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000075be34c1. 00000400:00000200:0.0:1784949210.551752:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:0.0:1784949210.551754:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:0.0:1784949210.551757:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949210.551763:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949210.551773:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.551782:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949210.551784:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551786:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949210.551788:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949210.551790:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949210.551792:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949210.551795:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949210.551801:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=77 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949210.551804:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=77 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949210.551807:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949210.551809:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949210.551811:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949210.551815:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949210.551824:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.551828:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949210.551830:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949210.551832:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949210.551834:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949210.551837:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.551845:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949210.551847:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949210.551849:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949210.551852:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949210.551860:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006a448376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289681792:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949210.551865:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551867:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949210.551870:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006a448376 x1871651289681792/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949226 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949210.551878:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949210.551880:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949210.551881:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949210.551885:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949210.551886:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949210.551888:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949210.551890:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949210.551892:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000009dcd7bb6. 02000000:00000001:0.0:1784949210.551895:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949210.551896:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949210.551898:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006a448376. 00000100:00000001:0.0:1784949210.551901:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949210.551941:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551945:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949210.551951:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949210.551954:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949210.551957:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949210.551961:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949210.592000:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306149931, at 4306149932 00000100:00000040:3.0:1784949210.592013:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306149931, at 4306149932 00000100:00000010:0.0:1784949210.592017:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00000100:00000040:1.0:1784949210.592029:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306149931, at 4306149932 00000100:00000010:3.0:1784949210.592032:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f68fd7be. 00000100:00000010:1.0:1784949210.592046:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000abbd7d53. 00000100:00000040:2.0:1784949210.592060:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306149931, at 4306149932 00000100:00000010:2.0:1784949210.592078:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000800:00000100:1.0:1784949210.740139:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000800:00000010:1.0:1784949210.740147:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e1f7df0e. 00000400:00000200:1.0:1784949210.740158:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949210.740166:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949210.740180:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949210.740186:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:1.0:1784949210.740191:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000010:3.0:1784949210.740198:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000000df08a31. 00000400:00000200:1.0:1784949210.740200:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:3.0:1784949210.740266:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dbce179e. 00000800:00000100:1.0:1784949210.740275:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000400:00000010:3.0:1784949210.740276:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000800:00000010:1.0:1784949210.740285:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000313a63a1. 00000400:00000200:1.0:1784949210.740291:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949210.740293:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e69d59cc. 00000400:00000200:1.0:1784949210.740303:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949210.740311:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949210.740311:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949210.740319:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11483 health :800 00000400:00000200:1.0:1784949210.740332:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949210.740334:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2861 00000400:00000200:1.0:1784949210.740342:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:3.0:1784949210.740346:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000001:0.0:1784949210.740349:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949210.740355:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949210.740358:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000400:00000200:3.0:1784949210.740359:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000010:0.0:1784949210.740362:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000400:00000200:3.0:1784949210.740370:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3768:3768:234:4294967295] 192.168.202.137@tcp LPNI seq info [2405:2405:8:4294967295] 00000400:00000200:0.0:1784949210.740373:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949210.740383:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949210.740392:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949210.740400:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949210.740404:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000800:00000010:3.0:1784949210.740411:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000400:00000200:0.0:1784949210.740418:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:0.0:1784949210.740426:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000859865aa. 00000400:00000200:0.0:1784949210.740440:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:0.0:1784949210.740445:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:0.0:1784949210.740452:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:0.0:1784949210.740458:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2861 00000400:00000001:0.0:1784949210.740461:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949210.740463:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949210.740466:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000010:0.0:1784949210.740468:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000400:00000200:0.0:1784949210.740473:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949210.740476:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:0.0:1784949210.740479:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f0621080. 00000400:00000200:0.0:1784949210.740483:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:0.0:1784949210.740485:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:0.0:1784949210.740488:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000800:00000200:3.0:1784949210.740561:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949210.740572:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949210.740577:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:2.0:1784949210.740748:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949210.740762:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000400:00000200:2.0:1784949210.740769:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949210.740776:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949210.740782:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949210.740786:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949210.740800:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949210.740813:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949210.740817:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949210.740822:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949210.740832:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949210.740847:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949210.740859:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949210.740870:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949210.740876:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3769:3769:235:4294967295] 192.168.202.137@tcp LPNI seq info [2406:2406:8:4294967295] 00000400:00000200:2.0:1784949210.740887:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949210.740892:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949210.740897:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 00000800:00000200:2.0:1784949210.740940:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949210.740949:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949210.740953:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:2.0:1784949210.741055:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949210.741059:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000400:00000200:2.0:1784949210.741066:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949210.741073:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949210.741079:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949210.741088:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949210.741092:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949210.741102:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949210.741108:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949210.741113:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949210.741123:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949210.741134:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x929d 00000800:00000001:2.0:1784949210.741139:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949210.741145:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949210.741147:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949210.741151:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949210.741155:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:2.0:1784949210.741158:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000400:00000200:2.0:1784949210.741161:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949210.741164:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000000df08a31. 00000400:00000010:2.0:1784949210.741168:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dbce179e. 00000800:00000001:2.0:1784949210.741176:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949210.741179:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949211.380045:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949211.380054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949211.380063:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949211.380067:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949211.380070:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949211.380074:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949211.380078:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949211.380081:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949211.380083:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949211.380087:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949211.380338:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949211.380344:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949211.380349:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949211.572021:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949211.572029:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784949211.572044:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949210/real 0] req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949211 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.572063:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.572066:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949211.572068:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949211.572072:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949211 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.572082:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.572092:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949211.572094:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.572098:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949211.572101:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949211.572104:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949211.572108:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949211 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.572117:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949211.572122:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.572124:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949211.572126:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.572129:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784949211.572132:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784949211.572136:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784949211.572165:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:1.0:1784949211.572168:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784949211.572172:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949211 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949211.572182:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289681664:1.2.3.4@tcp:8: 00000100:00000001:1.0:1784949211.572187:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.572190:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949211.572193:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008223e251 x1871651289681664/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949211 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.572239:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1784949211.572243:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949211.572245:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:1.0:1784949211.572249:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949211.572254:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000008d04782c. 02000000:00000001:1.0:1784949211.572262:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949211.572265:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949211.572267:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008223e251. 00000100:00000001:1.0:1784949211.572273:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949211.572275:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.572277:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.572282:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949211.572284:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.572288:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949211.572290:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949211.572305:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949211.572319:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949211.572330:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000074c559e1. 00000020:00000040:3.0:1784949211.572339:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949211.572346:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949211.572349:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949211.572355:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949211.572364:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:3.0:1784949211.572373:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949211.572381:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949211.572395:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949211.572398:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949211.572404:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000074c559e1] to pc [ptlrpcd_00_01+1] req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949211.572450:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949211.572453:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949211.572457:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1784949211.572460:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1784949211.572463:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1784949211.572469:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784949211.572475:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11458 00000100:00000001:1.0:1784949211.572486:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949211.572488:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:1.0:1784949211.572494:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1784949211.572498:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11482 00000100:00000001:1.0:1784949211.572501:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949211.572512:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949211.572522:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:1.0:1784949211.572534:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1784949211.572535:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:1.0:1784949211.572542:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949211.572544:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1784949211.572551:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949211.572556:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949211.572556:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:1.0:1784949211.572558:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949211.572566:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.572568:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949211.572569:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949211.572582:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289681920:0@lo:400: 00000100:00000040:3.0:1784949211.572583:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949211.572594:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:1.0:1784949211.572596:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1784949211.572603:0:335217:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784949211.572608:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1784949211.572609:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:1.0:1784949211.572615:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949211.572617:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1784949211.572623:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1784949211.572625:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:3.0:1784949211.572631:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000010:1.0:1784949211.572633:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004520da55. 00000020:00000001:3.0:1784949211.572635:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:2.0:1784949211.572638:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 02000000:00000001:1.0:1784949211.572641:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949211.572647:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:3.0:1784949211.572652:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1784949211.572654:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000317b8d47. 00000400:00000200:2.0:1784949211.572655:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000040:3.0:1784949211.572662:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949211.572668:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:3.0:1784949211.572671:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:1.0:1784949211.572672:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000400:00000010:2.0:1784949211.572674:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000098dd29a0. 00000100:00080000:3.0:1784949211.572680:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000400:00000010:2.0:1784949211.572686:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000100:00000200:1.0:1784949211.572687:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289681920, portal 25 00000100:00080000:3.0:1784949211.572687:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:1.0:1784949211.572699:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949211.572701:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949211.572705:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fa5520c4. 00000100:00000001:1.0:1784949211.572706:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554956160 : -115544154595456 : ffff96e9c7b59f80) 02000000:00000001:3.0:1784949211.572707:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1784949211.572719:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949211.572724:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949211.572729:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 02000000:00000001:3.0:1784949211.572732:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.572742:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1784949211.572748:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.572750:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784949211.572757:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000040:1.0:1784949211.572764:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000010:3.0:1784949211.572767:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000400:00000010:1.0:1784949211.572772:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000400:00000200:2.0:1784949211.572775:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1784949211.572776:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:1.0:1784949211.572781:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289681920, offset 0 02000000:00000001:3.0:1784949211.572788:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1784949211.572792:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:3.0:1784949211.572794:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949211.572795:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949211.572804:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784949211.572805:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949211.572812:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1784949211.572816:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:3.0:1784949211.572818:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949211.572826:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3770:3770:235:4294967295] 1.2.3.4@tcp LPNI seq info [115:115:8:4294967295] 02000000:00000010:3.0:1784949211.572829:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000579609a4. 00000100:00000001:3.0:1784949211.572837:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949211.572839:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00100000:3.0:1784949211.572852:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949211.572862:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:2.0:1784949211.572889:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:3.0:1784949211.572890:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949211.572896:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949211.572897:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949211.572956:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_rcv+-1] req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949211.572958:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d000 00000800:00000010:2.0:1784949211.572959:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000400:00000200:1.0:1784949211.572982:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8891 [2] + 0 00000800:00000200:2.0:1784949211.572986:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:1.0:1784949211.573000:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000093b2e4e6. 00000800:00000200:2.0:1784949211.573001:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 5 secs later) 00000400:00000200:1.0:1784949211.573012:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000005456a5d6 00000800:00000200:2.1:1784949211.573015:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:3.0:1784949211.573022:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1784949211.573022:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 5 secs later) 00000400:00000200:1.0:1784949211.573025:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784949211.573040:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1784949211.573045:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000100:2.0:1784949211.573047:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784949211.573057:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:2.0:1784949211.573058:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:1.0:1784949211.573062:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005456a5d6 00000100:00000001:3.0:1784949211.573062:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949211.573065:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949211.573071:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000010:1.0:1784949211.573072:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000005456a5d6. 00000100:00000001:0.0:1784949211.573073:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1784949211.573076:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003492edb3. 00000100:00000040:0.0:1784949211.573088:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949211.573089:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949211.573090:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:2.0:1784949211.573091:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000200:1.0:1784949211.573098:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784949211.573106:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1784949211.573109:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000040:1.0:1784949211.573112:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000001eb61cc5 x1871651289681920 msgsize 224 02000000:00000001:3.0:1784949211.573115:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1784949211.573123:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 02000000:00000001:3.0:1784949211.573126:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949211.573141:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00100000:1.0:1784949211.573142:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:0.0:1784949211.573143:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:3.0:1784949211.573144:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:0.0:1784949211.573152:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949211.573153:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949211.573157:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949211.573159:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 02000000:00000001:0.0:1784949211.573169:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784949211.573170:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000001:1.0:1784949211.573173:0:337120:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:0.0:1784949211.573178:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949211.573180:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784949211.573189:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 02000000:00000001:0.0:1784949211.573192:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949211.573196:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000400:00000200:2.0:1784949211.573222:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000100:00000001:1.0:1784949211.573235:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:2.0:1784949211.573238:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1784949211.573239:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289682048:192.168.202.137@tcp:8: 00000400:00000001:2.0:1784949211.573245:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:1.0:1784949211.573247:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949211.573252:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000100:00000001:0.0:1784949211.573262:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:2.0:1784949211.573262:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 02000000:00000001:0.0:1784949211.573274:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1784949211.573277:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 02000000:00000001:0.0:1784949211.573283:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949211.573293:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000001:0.0:1784949211.573297:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:2.0:1784949211.573305:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000098dd29a0. 00000100:00000001:1.0:1784949211.573311:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:0.0:1784949211.573313:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000000b08bf97. 00000400:00000200:2.0:1784949211.573319:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:3.0:1784949211.573322:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1784949211.573324:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949211.573325:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.573328:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949211.573334:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000400:00000010:0.0:1784949211.573341:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000075be34c1. 00000100:00000040:3.0:1784949211.573348:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003492edb3] to pc [ptlrpcd_00_02+2] req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949211.573355:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000400:00000010:0.0:1784949211.573360:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f3654321. 00000100:00000200:0.0:1784949211.573375:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289682048, portal 4 00000100:00000001:1.0:1784949211.573386:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949211.573388:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949211.573398:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000100:00000001:1.0:1784949211.573398:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949211.573409:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.573411:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:0.0:1784949211.573420:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.573422:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306150955 00000100:00100000:2.0:1784949211.573423:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289681920 00000100:00000001:3.0:1784949211.573423:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949211.573433:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000010:1.0:1784949211.573438:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000040:3.0:1784949211.573439:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11488) 00000100:00000001:2.0:1784949211.573441:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949211.573450:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.573457:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1784949211.573461:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949211.573465:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1784949211.573470:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289681920 00000100:00000001:1.0:1784949211.573472:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306150955 00000020:00000001:2.0:1784949211.573478:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:0.0:1784949211.573482:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949211.573483:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1784949211.573485:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:1.0:1784949211.573486:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00100000:3.0:1784949211.573493:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000020:00000001:2.0:1784949211.573494:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784949211.573496:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000236111b7. 00000100:00000001:1.0:1784949211.573497:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949211.573503:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784949211.573504:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000200:0.0:1784949211.573507:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289682048, offset 0 00000100:00000001:1.0:1784949211.573509:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306150955 00000020:00000001:2.0:1784949211.573517:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949211.573519:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1784949211.573522:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:3.0:1784949211.573526:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949211.573528:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784949211.573531:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949211.573534:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949211.573537:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949211.573543:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949211.573545:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949211.573546:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306150955 00000100:00000001:3.0:1784949211.573552:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000010:2.0:1784949211.573553:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000400:00000200:0.0:1784949211.573554:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000010:1.0:1784949211.573557:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000040:3.0:1784949211.573564:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949211.573565:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949211.573569:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000200:0.0:1784949211.573570:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:2.0:1784949211.573582:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00000001:1.0:1784949211.573584:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949211.573586:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949211.573598:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949211.573602:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:2.0:1784949211.573603:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000098dd29a0. 00000100:00000001:1.0:1784949211.573610:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.573611:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784949211.573616:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949211.573618:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949211.573620:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000059fdf11b. 00000100:00000001:1.0:1784949211.573621:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949211.573628:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784949211.573633:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3771:3771:235:4294967295] 192.168.202.137@tcp LPNI seq info [2407:2407:8:4294967295] 02000000:00000001:3.0:1784949211.573636:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949211.573638:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784949211.573640:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784949211.573647:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 02000000:00000001:3.0:1784949211.573648:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.573661:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949211.573666:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289682176:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949211.573668:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784949211.573668:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:0.0:1784949211.573670:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949211.573676:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.573684:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949211.573689:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949211.573690:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.573692:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784949211.573696:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000800:00000200:0.0:1784949211.573701:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949211.573703:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.573705:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949211.573710:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008f5595bf. 02000000:00000001:3.0:1784949211.573711:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949211.573714:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949211.573721:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949211.573722:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000035b54a67. 00000100:00000001:2.0:1784949211.573726:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949211.573730:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:3.0:1784949211.573732:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949211.573744:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000100:00000001:2.0:1784949211.573747:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949211.573753:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000467f058f. 00000800:00000200:0.0:1784949211.573757:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000200:3.0:1784949211.573763:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289682176, portal 10 00000800:00000200:0.0:1784949211.573767:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f5595bf type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1784949211.573770:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949211.573775:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949211.573778:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554958848 : -115544154592768 : ffff96e9c7b5aa00) 00000100:00000001:2.0:1784949211.573781:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949211.573791:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949211.573794:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949211.573798:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.573799:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949211.573799:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 802 00000100:00000040:2.0:1784949211.573815:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000040:0.0:1784949211.573817:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000001:1.0:1784949211.573817:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.573818:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949211.573824:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741505616 : -115539968046000 : ffff96eac13f4c50) 00000100:00000001:1.0:1784949211.573827:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949211.573831:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306150955 00000100:00000040:3.0:1784949211.573834:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949211.573843:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000040:2.0:1784949211.573844:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001eb61cc5 x1871651289681920/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:117/0 lens 224/0 e 0 to 0 dl 1784949222 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949211.573846:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306150955 00000100:00000200:3.0:1784949211.573852:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289682176, offset 0 00000100:00000001:0.0:1784949211.573860:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306150955 00000100:00000001:0.0:1784949211.573875:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306150955 00000400:00000200:3.0:1784949211.573878:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949211.573887:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306150955 00000100:00000001:2.0:1784949211.573889:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949211.573893:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306150955 00000020:00000010:2.0:1784949211.573898:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000400:00000200:3.0:1784949211.573899:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949211.573901:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306150955 00000100:00000001:0.0:1784949211.573950:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306150955 00000400:00000200:3.0:1784949211.573955:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784949211.573958:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949211.573964:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949211.573965:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949211.573967:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949211.573978:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:2.0:1784949211.573979:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00000001:0.0:1784949211.573981:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949211.573989:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784949211.573990:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949211.574001:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001eb61cc5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289681920:12345-0@lo:400: 00000400:00000200:3.0:1784949211.574002:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3772:3772:234:4294967295] 192.168.202.137@tcp LPNI seq info [2408:2408:7:4294967295] 00000100:00000001:0.0:1784949211.574005:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949211.574012:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000200:2.0:1784949211.574019:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289681920 00000100:00000001:0.0:1784949211.574024:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784949211.574027:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949211.574029:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949211.574036:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949211.574038:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000200:1.0:1784949211.574039:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949211.574044:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949211.574046:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:2.0:1784949211.574048:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949211.574049:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008f5595bf. 00000020:00000001:2.0:1784949211.574062:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:1.0:1784949211.574065:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949211.574069:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:3.0:1784949211.574071:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784949211.574081:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000010:3.0:1784949211.574086:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000001:0.0:1784949211.574087:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949211.574088:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949211.574088:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949211.574101:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:1.0:1784949211.574103:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000236111b7 00000800:00000001:0.0:1784949211.574109:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949211.574109:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000200:3.0:1784949211.574110:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:1.0:1784949211.574111:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000236111b7. 00000020:00000001:2.0:1784949211.574118:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949211.574121:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949211.574123:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1784949211.574128:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:0.0:1784949211.574131:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949211.574132:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:2.0:1784949211.574136:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000200:1.0:1784949211.574139:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949211.574144:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000200:3.0:1784949211.574146:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 02000000:00000010:2.0:1784949211.574155:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006f496819. 00000400:00000200:0.0:1784949211.574156:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:2.0:1784949211.574167:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.574173:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.574176:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949211.574185:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2220 usecs (3 jiffies) 00000020:00000001:2.0:1784949211.574188:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949211.574189:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643d080 00000100:00000001:3.0:1784949211.574220:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306150917 00000100:00000001:1.0:1784949211.574222:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000040:2.0:1784949211.574224:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289681920 00000100:00000040:1.0:1784949211.574234:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949236 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949211.574236:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949211.574236:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306150955 00000400:00000100:0.0:1784949211.574241:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289682048 offset 0 length 520: 4 00010000:00000200:2.0:1784949211.574251:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001eb61cc5 x1871651289681920/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:117/0 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949211.574252:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306150917 00000400:00000200:0.0:1784949211.574263:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000169545f3 is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:3.0:1784949211.574269:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306150955 00000400:00000200:0.0:1784949211.574279:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:3.0:1784949211.574286:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306150917 00000100:00000001:3.0:1784949211.574302:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306150955 00000100:00000001:1.0:1784949211.574303:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949211.574306:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784949211.574307:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949211.574314:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:2.0:1784949211.574318:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.574318:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306150917 00000100:00001000:2.0:1784949211.574326:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=83 v=5 (1 1 1 1) 00000800:00000001:0.0:1784949211.574331:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.574331:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306150955 00000100:00000001:2.0:1784949211.574335:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949211.574344:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949211.574345:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000001:0.0:1784949211.574346:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949211.574355:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949211.574356:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949211.574365:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784949211.574367:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949211.574370:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.574374:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.574377:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949211.574385:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949211.574392:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000100:00000200:2.0:1784949211.574398:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289681920, offset 224 00000400:00000200:2.0:1784949211.574404:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949211.574414:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949211.574428:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d000 00000800:00000001:0.0:1784949211.574434:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949211.574441:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x92a1 [1] + 224 00000800:00000001:0.0:1784949211.574449:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949211.574451:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949211.574455:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949211.574462:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:0.0:1784949211.574465:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949211.574468:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949211.574470:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000010:1.0:1784949211.574477:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:0.0:1784949211.574482:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784949211.574483:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949211.574488:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949211.574509:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d100 00000400:00000200:1.0:1784949211.574511:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949211.574521:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949211.574522:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949211.574534:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 175312 00000400:00000200:1.0:1784949211.574551:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000800:00000001:0.0:1784949211.574554:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:1.0:1784949211.574557:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:1.0:1784949211.574563:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949211.574571:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949211.574575:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949211.574576:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949211.574586:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949211.574590:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000100:00000001:1.0:1784949211.574594:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:2.0:1784949211.574597:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000400:00000200:0.0:1784949211.574605:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949211.574606:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:1.0:1784949211.574607:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949211.574613:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949211.574614:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949211.574621:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949211.574622:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949211.574626:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949211.574630:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000001:3.0:1784949211.574635:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784949211.574642:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:3.0:1784949211.574644:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784949211.574645:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949211.574647:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000064dc7d78. 00010000:00000001:2.0:1784949211.574649:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:3.0:1784949211.574653:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000317b8d47. 00000100:00000001:1.0:1784949211.574656:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1784949211.574658:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949211.574659:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000064dc7d78 x1871651289682176 msgsize 224 00000400:00000200:3.0:1784949211.574665:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000020:00000001:2.0:1784949211.574668:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:3.0:1784949211.574672:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000100:00000040:2.0:1784949211.574681:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001eb61cc5 x1871651289681920/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:117/0 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949211.574682:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784949211.574683:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949211.574697:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949211.574703:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784949211.574717:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784949211.574722:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001eb61cc5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289681920:12345-0@lo:400: Request processed in 824us (1600us total) trans 0 rc 0/0 00000100:00000200:3.0:1784949211.574727:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949211.574728:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.574734:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949211.574742:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289682176 00000100:00100000:2.0:1784949211.574744:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 802 02000000:00000001:1.0:1784949211.574752:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:2.0:1784949211.574757:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949211.574759:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949211.574767:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.574768:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949211.574769:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949211.574779:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949211.574780:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949211.574781:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949211.574791:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:2.0:1784949211.574792:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000059fdf11b. 00000100:00100000:1.0:1784949211.574793:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289682176 02000000:00000001:3.0:1784949211.574798:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:1.0:1784949211.574803:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949211.574805:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:2.0:1784949211.574806:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000040:1.0:1784949211.574808:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:3.0:1784949211.574810:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949211.574816:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1784949211.574819:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949211.574820:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000098dd29a0. 00000020:00000040:1.0:1784949211.574822:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949211.574832:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00001000:3.0:1784949211.574835:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=83 v=5 (5 5 5 5) 00000020:00000040:2.0:1784949211.574836:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:1.0:1784949211.574837:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:2.0:1784949211.574848:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949211.574851:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00001000:3.0:1784949211.574852:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=83 v=5 (1 1 1 1) 02000000:00000010:2.0:1784949211.574853:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006f496819. 00000020:00000010:1.0:1784949211.574861:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 02000000:00000001:2.0:1784949211.574864:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949211.574865:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949211.574869:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949211.574873:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.574874:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949211.574875:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00010000:00000001:3.0:1784949211.574888:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000200:2.0:1784949211.574891:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000010:1.0:1784949211.574892:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00010000:00000001:3.0:1784949211.574892:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949211.574944:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a86c77e2. 00000100:00000001:3.0:1784949211.574945:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949211.574949:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000056a1a23f. 00000400:00000010:2.0:1784949211.574961:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009a6bd01a. 00000020:00000010:1.0:1784949211.574962:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:3.0:1784949211.574962:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949211.574981:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949211.574982:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949211.574990:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949211.574991:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949211.574995:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949211.574997:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.575000:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949211.575007:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.575010:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949211.575016:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949211.575021:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949211.575027:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000074c559e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289681920:0@lo:400: 00000100:00000001:3.0:1784949211.575035:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575037:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949211.575040:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000074c559e1 x1871651289681920/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.575042:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575056:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949211.575061:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1784949211.575062:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000010:3.0:1784949211.575065:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004520da55. 00000100:00000001:1.0:1784949211.575067:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:3.0:1784949211.575074:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1784949211.575081:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 778 00000020:00000001:3.0:1784949211.575082:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949211.575087:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000040:1.0:1784949211.575090:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000020:00000001:3.0:1784949211.575095:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:1.0:1784949211.575097:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554968960 : -115544154582656 : ffff96e9c7b5d180) 02000000:00000010:3.0:1784949211.575100:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:3.0:1784949211.575107:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1784949211.575108:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000064dc7d78 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:117/0 lens 224/0 e 0 to 0 dl 1784949222 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949211.575110:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949211.575116:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000074c559e1. 00000100:00000001:3.0:1784949211.575126:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949211.575130:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949211.575131:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949211.575135:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949211.575139:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:3.0:1784949211.575143:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949211.575144:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949211.575148:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949211.575150:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:3.0:1784949211.575155:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949211.575161:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949211.575162:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000064dc7d78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289682176:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949211.575173:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289682176 00000020:00000001:1.0:1784949211.575176:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949211.575179:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949211.575182:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949211.575185:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949211.575188:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949211.575191:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949211.575195:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949211.575221:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949211.575224:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949211.575227:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949211.575229:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949211.575233:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949211.575236:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949211.575240:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008d04782c. 02000000:00000001:1.0:1784949211.575242:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.575245:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949211.575247:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949211.575250:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289682176 00010000:00000001:1.0:1784949211.575252:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949211.575258:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000064dc7d78 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:117/0 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949211.575268:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949211.575270:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949211.575273:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=78 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949211.575278:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949211.575280:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949211.575283:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949211.575286:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949211.575288:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949211.575290:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949211.575293:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949211.575296:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00000200:1.0:1784949211.575299:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289682176, offset 224 00000400:00000200:1.0:1784949211.575304:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949211.575313:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949211.575319:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3773:3773:235:4294967295] 192.168.202.137@tcp LPNI seq info [2409:2409:8:4294967295] 00000400:00000200:1.0:1784949211.575330:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949211.575336:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949211.575341:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a1bdc144. 00000800:00000200:1.0:1784949211.575346:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949211.575353:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949211.575356:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a1bdc144 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949211.575376:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949211.575380:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949211.575382:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949211.575384:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949211.575387:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949211.575391:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000064dc7d78 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:117/0 lens 224/224 e 0 to 0 dl 1784949222 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949211.575402:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000064dc7d78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289682176:12345-192.168.202.137@tcp:400: Request processed in 271us (742us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949211.575411:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 778 00000100:00000040:1.0:1784949211.575415:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949211.575418:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949211.575419:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949211.575423:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000056a1a23f. 00000020:00000010:1.0:1784949211.575427:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949211.575432:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000020:00000040:1.0:1784949211.575436:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949211.575440:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949211.575547:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949211.575556:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a1bdc144. 00000400:00000200:3.0:1784949211.575567:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949211.575579:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949211.575583:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949211.575588:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:3.0:1784949211.575595:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000800:00000001:1.0:1784949211.575596:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949211.575605:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949211.575605:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784949211.575609:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575612:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949211.575623:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949211.575637:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d100 00000400:00000200:1.0:1784949211.575645:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x92b5 [1] + 224 00000800:00000001:1.0:1784949211.575652:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949211.575658:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949211.575661:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949211.575665:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949211.575671:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949211.575675:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949211.575686:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949211.575704:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949211.575712:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949211.575716:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575738:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949211.575743:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949211.575749:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949211.575751:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949211.575754:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000400:00000200:3.0:1784949211.575762:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000400:00000010:3.0:1784949211.575764:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000100:00000001:3.0:1784949211.575767:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949211.575774:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949211.575788:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949211.575798:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949211.575800:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575803:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949211.575805:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949211.575807:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949211.575809:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949211.575812:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949211.575819:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=78 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949211.575823:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=78 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949211.575826:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949211.575828:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949211.575830:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949211.575834:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949211.575843:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.575847:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949211.575849:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949211.575851:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949211.575852:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949211.575855:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949211.575864:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949211.575867:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949211.575869:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949211.575873:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949211.575882:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003492edb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289682176:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949211.575887:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575890:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949211.575892:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003492edb3 x1871651289682176/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949211.575899:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949211.575901:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949211.575931:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000035b54a67. 02000000:00000001:3.0:1784949211.575938:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949211.575940:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949211.575942:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949211.575945:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949211.575947:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:3.0:1784949211.575950:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949211.575952:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949211.575953:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003492edb3. 00000100:00000001:3.0:1784949211.575957:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949211.575959:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575961:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949211.575966:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949211.575968:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949211.575971:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949211.575973:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949211.616030:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306150955, at 4306150956 00000100:00000040:2.0:1784949211.616035:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306150955, at 4306150956 00000100:00000040:0.0:1784949211.616040:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306150955, at 4306150956 00000100:00000010:1.0:1784949211.616045:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:2.0:1784949211.616051:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:0.0:1784949211.616055:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:3.0:1784949211.616111:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306150955, at 4306150956 00000100:00000010:3.0:1784949211.616124:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00010000:00000001:2.0:1784949212.404047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949212.404058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949212.404069:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949212.404072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949212.404158:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949212.404164:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949212.404169:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949213.428066:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949213.428076:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949213.428084:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949213.428086:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949213.428235:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949213.428242:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949213.428246:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949214.452048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949214.452057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949214.452065:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949214.452067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949214.452231:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949214.452238:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949214.452244:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949215.476002:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949215.476009:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949215.476014:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949215.476015:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949215.476090:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949215.476093:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949215.476095:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949216.499985:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949216.499992:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949216.499996:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949216.499998:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949216.500081:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949216.500085:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949216.500087:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784949216.692040:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949216.692041:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949216.692055:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949216.692057:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949216.692061:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949216.692067:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784949216.692069:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008223e251. 00000100:00000001:0.0:1784949216.692071:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949216.692076:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.692082:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949216.692082:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949216.692094:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949216.692098:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949216.692104:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949216.692114:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002ddfcc2d. 00000100:00000001:1.0:1784949216.692122:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949216.692133:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949216.692146:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949216.692150:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949216.692193:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008223e251] to pc [ptlrpcd_00_03+3] req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949216.692225:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949216.692228:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949216.692233:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949216.692236:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949216.692239:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949216.692243:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009c295fd6. 00000020:00000040:1.0:1784949216.692246:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949216.692249:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949216.692252:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949216.692252:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949216.692256:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.692259:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1784949216.692263:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c158c1d0. 00000100:00000001:3.0:1784949216.692264:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949216.692273:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949216.692276:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949216.692281:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949216.692284:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949216.692291:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949216.692322:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949216.692322:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949216.692326:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.692328:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949216.692334:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c295fd6] to pc [ptlrpcd_00_00+0] req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949216.692336:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949216.692341:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949216.692348:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949216.692357:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289682304:0@lo:400: 00000100:00000001:3.0:1784949216.692363:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949216.692370:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949216.692372:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949216.692375:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949216.692376:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1784949216.692385:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11493) 02000000:00000010:3.0:1784949216.692386:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000035b54a67. 02000000:00000001:3.0:1784949216.692392:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949216.692400:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000100:00000001:0.0:1784949216.692405:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949216.692409:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000467f058f. 00000100:00000001:0.0:1784949216.692411:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949216.692417:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000200:3.0:1784949216.692422:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289682304, portal 25 00000100:00000001:2.0:1784949216.692426:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949216.692427:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949216.692429:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949216.692434:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637810048 : -115544071741568 : ffff96e9cca5df80) 00000100:00000001:2.0:1784949216.692435:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949216.692447:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949216.692459:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949216.692464:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.692466:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949216.692469:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1784949216.692474:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784949216.692476:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784949216.692478:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 02000000:00000001:0.0:1784949216.692482:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949216.692485:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289682304, offset 0 00000100:00100000:0.0:1784949216.692489:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000009c295fd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289682432:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949216.692495:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784949216.692496:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:0.0:1784949216.692499:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949216.692503:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949216.692505:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949216.692510:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949216.692515:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949216.692516:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:0.0:1784949216.692523:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000094ceb1ba. 00000400:00000010:0.0:1784949216.692532:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000082a1e73a. 00000400:00000200:3.0:1784949216.692538:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d180 00000100:00000200:0.0:1784949216.692540:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289682432, portal 10 00000100:00000001:0.0:1784949216.692545:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949216.692546:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637814528 : -115544071737088 : ffff96e9cca5f100) 00000400:00000200:3.0:1784949216.692551:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x88ad [2] + 0 00000400:00000010:3.0:1784949216.692557:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f237c65a. 00000100:00000040:0.0:1784949216.692560:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949216.692566:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000b1caac62 00000100:00000001:0.0:1784949216.692573:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949216.692577:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949216.692578:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:0.0:1784949216.692581:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000089c8262f. 00000400:00000200:3.0:1784949216.692587:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:0.0:1784949216.692589:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289682432, offset 0 00000400:00000200:3.0:1784949216.692596:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b1caac62 00000400:00000010:3.0:1784949216.692601:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000b1caac62. 00000100:00000001:3.0:1784949216.692607:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949216.692612:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1784949216.692613:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949216.692620:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000074ed294d x1871651289682304 msgsize 224 00000100:00100000:3.0:1784949216.692629:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:0.0:1784949216.692633:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:3.0:1784949216.692634:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:0.0:1784949216.692644:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949216.692650:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949216.692654:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784949216.692655:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949216.692660:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949216.692665:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3774:3774:235:4294967295] 192.168.202.137@tcp LPNI seq info [2410:2410:8:4294967295] 00000400:00000200:0.0:1784949216.692678:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949216.692685:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949216.692690:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003ed79484. 00000400:00000200:3.0:1784949216.692692:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000400:00000010:3.0:1784949216.692696:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000800:00000200:0.0:1784949216.692701:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949216.692703:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949216.692710:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949216.692717:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949216.692723:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1784949216.692725:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289682304 00000800:00000200:0.0:1784949216.692728:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:3.0:1784949216.692731:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949216.692734:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000200:0.0:1784949216.692736:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003ed79484 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949216.692742:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949216.692749:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.692753:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.692756:0:337122:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784949216.692759:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.692763:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.692763:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949216.692768:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289682304 00000100:00000001:3.0:1784949216.692774:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306156074 00000020:00000001:2.0:1784949216.692775:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:0.0:1784949216.692776:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2221 usecs (3 jiffies) 00000020:00000040:2.0:1784949216.692784:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949216.692785:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306156035 00000100:00000010:3.0:1784949216.692787:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000020:00000001:2.0:1784949216.692794:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949216.692800:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949216.692802:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1784949216.692808:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:2.0:1784949216.692815:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949216.692816:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306156074 00000100:00000001:0.0:1784949216.692817:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306156074 00000100:00000001:2.0:1784949216.692822:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949216.692825:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306156035 00000100:00000010:3.0:1784949216.692831:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000020:00000010:2.0:1784949216.692834:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000001:3.0:1784949216.692840:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949216.692842:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949216.692850:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00000001:0.0:1784949216.692852:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306156074 00000100:00000001:3.0:1784949216.692852:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306156074 00000100:00000001:0.0:1784949216.692860:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306156035 00000020:00000010:2.0:1784949216.692864:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000098dd29a0. 00000100:00000010:3.0:1784949216.692866:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000020:00000010:2.0:1784949216.692876:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000059fdf11b. 00000100:00000001:3.0:1784949216.692877:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949216.692888:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306156074 00000020:00000010:2.0:1784949216.692889:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784949216.692890:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306156074 00000100:00000001:0.0:1784949216.692893:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306156035 00000100:00000010:3.0:1784949216.692933:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:3.0:1784949216.692939:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949216.692944:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306156074 00000100:00000001:3.0:1784949216.692944:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949216.692950:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.692952:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949216.692954:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949216.692956:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.692957:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.692965:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949216.692966:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949216.692967:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.692973:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949216.692975:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949216.692978:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949216.692979:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.692983:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:1.0:1784949216.692983:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949216.692984:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949216.692991:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000010:1.0:1784949216.692993:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003ed79484. 00000100:00000001:2.0:1784949216.692994:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.692994:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949216.692999:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949216.693000:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949216.693005:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949216.693006:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949216.693007:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.693011:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.693018:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.693019:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.693019:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949216.693027:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949216.693037:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000089c8262f 00000400:00000010:1.0:1784949216.693041:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000089c8262f. 00000800:00000001:3.0:1784949216.693043:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949216.693048:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949216.693049:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949216.693056:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:3.0:1784949216.693057:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949216.693059:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949216.693064:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949216.693071:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949216.693072:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:2.0:1784949216.693072:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 803 00000100:00000001:0.0:1784949216.693078:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:2.0:1784949216.693086:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949216.693087:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949216.693093:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949216.693096:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.693098:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741507664 : -115539968043952 : ffff96eac13f5450) 00000100:00000001:1.0:1784949216.693104:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949216.693119:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949216.693120:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000074ed294d x1871651289682304/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:122/0 lens 224/0 e 0 to 0 dl 1784949227 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949216.693120:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d200 00000400:00000200:3.0:1784949216.693140:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 175536 00000100:00000001:1.0:1784949216.693181:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949216.693183:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949216.693188:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949216.693191:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949216.693200:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000800:00000001:3.0:1784949216.693201:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949216.693208:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949216.693210:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949216.693219:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:2.0:1784949216.693220:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00000001:3.0:1784949216.693228:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949216.693232:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00100000:2.0:1784949216.693235:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000074ed294d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289682304:12345-0@lo:400: 00000100:00000010:3.0:1784949216.693238:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003492edb3. 00000100:00000040:3.0:1784949216.693244:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000003492edb3 x1871651289682432 msgsize 224 00000100:00000200:2.0:1784949216.693251:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289682304 00000100:00100000:3.0:1784949216.693257:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:2.0:1784949216.693261:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949216.693268:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949216.693272:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949216.693275:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784949216.693276:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:2.0:1784949216.693280:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000001:3.0:1784949216.693286:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949216.693288:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949216.693291:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949216.693292:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949216.693298:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949216.693302:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949216.693305:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949216.693308:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949216.693312:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949216.693316:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949216.693322:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006f496819. 02000000:00000001:2.0:1784949216.693325:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.693328:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949216.693331:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949216.693334:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289682304 00000100:00000001:0.0:1784949216.693336:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:2.0:1784949216.693337:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00100000:0.0:1784949216.693344:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289682432 00010000:00000200:2.0:1784949216.693347:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000074ed294d x1871651289682304/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:122/0 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949216.693351:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949216.693356:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949216.693361:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949216.693364:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:0.0:1784949216.693368:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949216.693369:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949216.693379:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=88 v=5 (1 1 1 1) 00000100:00100000:0.0:1784949216.693380:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289682432 00000100:00000001:2.0:1784949216.693388:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1784949216.693389:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:2.0:1784949216.693394:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000040:0.0:1784949216.693395:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:2.0:1784949216.693399:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:0.0:1784949216.693401:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1784949216.693405:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000040:0.0:1784949216.693408:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:2.0:1784949216.693409:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.693414:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1784949216.693417:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:2.0:1784949216.693422:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:0.0:1784949216.693425:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000010:2.0:1784949216.693428:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000100:00000001:0.0:1784949216.693436:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:2.0:1784949216.693436:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289682304, offset 224 00000020:00000010:0.0:1784949216.693447:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000400:00000200:2.0:1784949216.693447:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949216.693455:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949216.693461:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000400:00000200:2.0:1784949216.693461:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784949216.693474:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000400:00000200:2.0:1784949216.693476:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d180 00000020:00000010:0.0:1784949216.693484:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cee92cdb. 00000400:00000200:2.0:1784949216.693485:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x92c9 [1] + 224 00000020:00000010:0.0:1784949216.693495:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000400:00000200:2.0:1784949216.693496:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949216.693504:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949216.693511:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784949216.693513:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000200:2.0:1784949216.693517:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949216.693520:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949216.693525:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949216.693531:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949216.693537:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949216.693541:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949216.693548:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949216.693552:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.693563:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949216.693570:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949216.693572:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:2.0:1784949216.693577:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000100:00000001:2.0:1784949216.693581:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949216.693583:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949216.693589:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949216.693590:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.693597:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:2.0:1784949216.693599:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784949216.693604:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949216.693608:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949216.693610:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 779 00000020:00000001:2.0:1784949216.693613:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949216.693620:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000040:2.0:1784949216.693622:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000074ed294d x1871651289682304/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:122/0 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949216.693629:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554958848 : -115544154592768 : ffff96e9c7b5aa00) 00000100:00000001:3.0:1784949216.693630:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949216.693636:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949216.693648:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:0.0:1784949216.693649:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003492edb3 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:122/0 lens 224/0 e 0 to 0 dl 1784949227 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949216.693650:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000074ed294d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289682304:12345-0@lo:400: Request processed in 455us (1023us total) trans 0 rc 0/0 00000400:00000001:3.0:1784949216.693653:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949216.693659:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000400:00000200:3.0:1784949216.693669:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000400:00000010:3.0:1784949216.693674:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000100:00100000:2.0:1784949216.693680:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 803 00000100:00000001:3.0:1784949216.693683:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1784949216.693691:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:3.0:1784949216.693692:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949216.693693:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949216.693697:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000010:0.0:1784949216.693703:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:2.0:1784949216.693704:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949216.693712:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000059fdf11b. 00000100:00000001:0.0:1784949216.693715:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784949216.693717:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949216.693721:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:0.0:1784949216.693727:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000020:00000010:2.0:1784949216.693730:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000098dd29a0. 00000020:00000040:2.0:1784949216.693741:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00100000:0.0:1784949216.693747:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003492edb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289682432:12345-192.168.202.137@tcp:400: 02000000:00000001:2.0:1784949216.693750:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949216.693750:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:2.0:1784949216.693756:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006f496819. 00000100:00000001:3.0:1784949216.693759:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949216.693760:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289682432 02000000:00000001:2.0:1784949216.693762:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:0.0:1784949216.693768:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949216.693769:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:2.0:1784949216.693770:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949216.693777:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:2.0:1784949216.693778:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:3.0:1784949216.693778:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:0.0:1784949216.693783:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.693785:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:2.0:1784949216.693787:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000037df9850. 00000020:00000001:0.0:1784949216.693790:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784949216.693792:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949216.693796:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000010:2.0:1784949216.693797:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b0e9c277. 02000000:00000001:3.0:1784949216.693799:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949216.693808:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00001000:3.0:1784949216.693812:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=88 v=5 (5 5 5 5) 00000020:00000001:0.0:1784949216.693815:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784949216.693822:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=88 v=5 (1 1 1 1) 00000020:00000001:0.0:1784949216.693823:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949216.693827:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949216.693832:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:0.0:1784949216.693833:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.693839:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949216.693840:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949216.693846:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00010000:00000001:3.0:1784949216.693848:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 02000000:00000001:0.0:1784949216.693852:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00010000:00000001:3.0:1784949216.693854:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949216.693859:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000021154f68. 00000100:00000001:3.0:1784949216.693862:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949216.693865:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949216.693872:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949216.693874:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949216.693880:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949216.693888:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289682432 00010000:00000001:0.0:1784949216.693896:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949216.693935:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003492edb3 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:122/0 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949216.693946:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1784949216.693955:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949216.693956:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00010000:00000001:0.0:1784949216.693963:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.693963:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784949216.693971:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=83 v=5 (1 1 1 1) 00000100:00000040:3.0:1784949216.693974:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949216.693983:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949216.693993:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949216.693999:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:3.0:1784949216.694001:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289682304:0@lo:400: 02000000:00000001:0.0:1784949216.694007:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949216.694013:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949216.694014:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949216.694018:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949216.694022:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949216.694029:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949216.694033:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008223e251 x1871651289682304/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949216.694038:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006eea6d76. 00000100:00000200:0.0:1784949216.694046:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289682432, offset 224 00000100:00000001:3.0:1784949216.694059:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:0.0:1784949216.694062:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949216.694065:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949216.694072:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000035b54a67. 00000400:00000200:0.0:1784949216.694079:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:3.0:1784949216.694084:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949216.694090:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:0.0:1784949216.694091:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3775:3775:235:4294967295] 192.168.202.137@tcp LPNI seq info [2411:2411:8:4294967295] 00000020:00000040:3.0:1784949216.694096:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949216.694103:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949216.694108:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002ddfcc2d. 02000000:00000001:3.0:1784949216.694133:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949216.694138:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:0.0:1784949216.694142:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000010:3.0:1784949216.694143:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008223e251. 00000100:00000001:3.0:1784949216.694153:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949216.694175:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949216.694179:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949216.694189:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949216.694197:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000062b4a223. 00000100:00000001:3.0:1784949216.694198:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949216.694205:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949216.694210:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949216.694221:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:0.0:1784949216.694226:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784949216.694228:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949216.694235:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000062b4a223 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949216.694258:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949216.694264:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949216.694268:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949216.694271:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949216.694274:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949216.694280:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003492edb3 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:122/0 lens 224/224 e 0 to 0 dl 1784949227 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949216.694296:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003492edb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289682432:12345-192.168.202.137@tcp:400: Request processed in 602us (1049us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949216.694309:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 779 00000100:00000040:0.0:1784949216.694314:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949216.694316:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949216.694318:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949216.694322:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cee92cdb. 00000020:00000010:0.0:1784949216.694325:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949216.694329:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000020:00000040:0.0:1784949216.694336:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949216.694338:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949216.694969:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949216.694985:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000062b4a223. 00000800:00000001:0.0:1784949216.695002:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949216.695003:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949216.695018:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949216.695027:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949216.695036:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949216.695037:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949216.695059:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949216.695061:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006eea6d76 00000400:00000010:1.0:1784949216.695071:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006eea6d76. 00000400:00000200:0.0:1784949216.695080:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d200 00000100:00000001:1.0:1784949216.695089:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:0.0:1784949216.695090:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x92d1 [1] + 224 00000100:00000001:1.0:1784949216.695100:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949216.695103:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949216.695110:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949216.695114:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949216.695121:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949216.695127:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949216.695132:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949216.695145:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949216.695198:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949216.695206:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949216.695210:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949216.695293:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949216.695299:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949216.695305:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949216.695307:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949216.695309:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000094ceb1ba. 00000400:00000200:2.0:1784949216.695316:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000082a1e73a 00000400:00000010:2.0:1784949216.695318:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000082a1e73a. 00000100:00000001:2.0:1784949216.695321:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949216.695327:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949216.695339:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949216.695348:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949216.695349:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949216.695352:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949216.695354:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949216.695356:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949216.695357:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949216.695360:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949216.695365:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=83 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949216.695369:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=83 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949216.695371:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949216.695373:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949216.695375:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949216.695378:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949216.695394:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.695398:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949216.695400:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949216.695402:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949216.695403:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949216.695406:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949216.695413:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949216.695415:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949216.695417:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949216.695420:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949216.695427:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000009c295fd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289682432:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949216.695431:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949216.695433:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949216.695435:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000009c295fd6 x1871651289682432/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949216.695442:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949216.695444:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949216.695446:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:2.0:1784949216.695450:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949216.695451:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949216.695452:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949216.695455:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949216.695456:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c158c1d0. 02000000:00000001:2.0:1784949216.695459:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949216.695460:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949216.695462:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000009c295fd6. 00000100:00000001:2.0:1784949216.695465:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949216.695466:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949216.695468:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949216.695473:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949216.695475:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949216.695477:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949216.695479:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949216.734960:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306156074, at 4306156075 00000100:00000010:0.0:1784949216.734969:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000040:2.0:1784949216.734975:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306156074, at 4306156075 00000100:00000010:2.0:1784949216.734983:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:1.0:1784949216.735232:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306156074, at 4306156075 00000100:00000010:1.0:1784949216.735250:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000040:3.0:1784949216.735990:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306156074, at 4306156076 00000100:00000010:3.0:1784949216.736000:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000400:00000010:3.0:1784949216.884006:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007864ed56. 00000400:00000010:3.0:1784949216.884024:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001542e92a. 00000400:00000010:3.0:1784949216.884028:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007f814b08. 00000400:00000010:3.0:1784949216.884035:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002026f8a1. 00000400:00000010:3.0:1784949216.884039:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000771e09fe. 00000400:00000010:3.0:1784949216.884041:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001bae6561. 00000400:00000010:3.0:1784949216.884043:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a0e11faa. 00000400:00000010:3.0:1784949216.884045:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000005c4dcc18. 00000400:00000010:3.0:1784949216.884050:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000fee11282. 00000400:00000010:3.0:1784949216.884057:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004d156103. 00000400:00000010:3.0:1784949216.884059:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000bfff640a. 00000400:00000010:3.0:1784949216.884061:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000098ebc0ae. 00000400:00000010:3.0:1784949216.884063:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000b1d42ed. 00000400:00000010:3.0:1784949216.884065:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e69b1edb. 00000400:00000010:3.0:1784949216.884067:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d5bb2c49. 00000400:00000010:3.0:1784949216.884069:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a2b37178. 00000400:00000010:3.0:1784949216.884071:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000024644ba3. 00000400:00000010:3.0:1784949216.884073:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000067f76b46. 00000400:00000010:3.0:1784949216.884074:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000087a70a9. 00000400:00000010:3.0:1784949216.884076:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f9462ca3. 00000400:00000010:3.0:1784949216.884078:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000fd8e815. 00000400:00000010:3.0:1784949216.884080:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002e599554. 00000400:00000010:3.0:1784949216.884081:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c12c1931. 00000400:00000010:3.0:1784949216.884083:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e42cbe57. 00000400:00000010:3.0:1784949216.884084:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000390dd32d. 00000400:00000010:3.0:1784949216.884086:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c732a1a8. 00000400:00000010:3.0:1784949216.884088:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009f5e55af. 00000400:00000010:3.0:1784949216.884090:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004fd5390b. 00000400:00000010:3.0:1784949216.884093:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008c288862. 00000400:00000010:3.0:1784949216.884095:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000012554f3a. 00000400:00000010:3.0:1784949216.884096:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ca3adacb. 00000400:00000010:3.0:1784949216.884098:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000325e9e7. 00000400:00000010:3.0:1784949216.884100:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ae557b6e. 00000400:00000010:3.0:1784949216.884102:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b9d99f33. 00000400:00000010:3.0:1784949216.884104:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000064d54234. 00000400:00000010:3.0:1784949216.884106:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004f0f2833. 00000400:00000010:3.0:1784949216.884108:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006aad92e0. 00000400:00000010:3.0:1784949216.884109:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a816c742. 00000400:00000010:3.0:1784949216.884111:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000074e9a075. 00000400:00000010:3.0:1784949216.884113:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e4a8defd. 00000400:00000010:3.0:1784949216.884114:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000775d3d92. 00000400:00000010:3.0:1784949216.884116:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000473e84ce. 00000400:00000010:3.0:1784949216.884118:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000062d84ae9. 00000400:00000010:3.0:1784949216.884120:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000060798c14. 00000400:00000010:3.0:1784949216.884121:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a7d88830. 00000400:00000010:3.0:1784949216.884123:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000049855afc. 00000400:00000010:3.0:1784949216.884125:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c3fd57c9. 00000400:00000010:3.0:1784949216.884127:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d54d9bc5. 00000400:00000010:3.0:1784949216.884129:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b23f11e5. 00000400:00000010:3.0:1784949216.884132:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000084f96ec7. 00000400:00000010:3.0:1784949216.884134:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009cb36717. 00000400:00000010:3.0:1784949216.884136:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ccf4a53f. 00000400:00000010:3.0:1784949216.884138:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000521edb81. 00000400:00000010:3.0:1784949216.884139:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cd70637b. 00000400:00000010:3.0:1784949216.884141:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000455b4195. 00000400:00000010:3.0:1784949216.884142:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b2848ca3. 00000400:00000010:3.0:1784949216.884144:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002085654e. 00000400:00000010:3.0:1784949216.884146:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000079caf10d. 00000400:00000010:3.0:1784949216.884148:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000029ca9b8d. 00000400:00000010:3.0:1784949216.884149:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000189c3332. 00000400:00000010:3.0:1784949216.884151:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000511e5266. 00000400:00000010:3.0:1784949216.884172:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c986c2b1. 00000400:00000010:3.0:1784949216.884176:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000034614846. 00000400:00000010:3.0:1784949216.884178:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003859b1c5. 00000400:00000010:3.0:1784949216.884184:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000061508776. 00000400:00000010:3.0:1784949216.884185:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001e82affc. 00000400:00000010:3.0:1784949216.884187:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e2f3909c. 00000400:00000010:3.0:1784949216.884189:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e07f0bcd. 00000400:00000010:3.0:1784949216.884190:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000185e599a. 00000400:00000010:3.0:1784949216.884203:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001aade4ec. 00000400:00000010:3.0:1784949216.884205:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b5ecaa25. 00000400:00000010:3.0:1784949216.884206:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000023b3e3f1. 00010000:00000001:2.0:1784949217.524039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949217.524049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949217.524057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949217.524059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949217.524252:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949217.524262:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949217.524269:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949217.716081:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949217.716099:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949217.716101:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949217.716107:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949217.716111:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949217.716117:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949217.716120:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949218.548082:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949218.548091:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949218.548100:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949218.548102:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949218.548218:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949218.548225:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949218.548230:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949218.740167:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949218.740184:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949218.740187:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949218.740194:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949218.740198:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949218.740204:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949218.740207:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949219.572042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949219.572051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949219.572059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949219.572061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949219.572221:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949219.572228:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949219.572233:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949219.764055:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949219.764073:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949219.764076:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949219.764082:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949219.764086:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949219.764091:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949219.764095:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949220.596043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949220.596056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949220.596067:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949220.596070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949220.596179:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949220.596187:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949220.596193:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949220.788024:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949220.788043:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949220.788046:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949220.788053:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949220.788057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949220.788061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949220.788065:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949220.980043:0:337111:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008ddad1b6] -> 12345-192.168.202.37@tcp (4) 00000800:00000200:1.0:1784949220.980059:0:337111:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.37@tcp ip 192.168.202.37:1021 00000800:00000200:1.0:1784949220.980066:0:337111:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fa9b7aef type 192, nob 24 niov 1 nkiov 0 00000800:00000200:3.0:1784949220.980244:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1784949221.620034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949221.620044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949221.620052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949221.620056:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949221.620060:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949221.620064:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949221.620068:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949221.620070:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949221.620073:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949221.620075:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949221.620219:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949221.620226:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949221.620231:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784949221.812032:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949221.812046:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949221.812060:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ee628dd6. 00000020:00000040:1.0:1784949221.812069:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949221.812077:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949221.812081:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949221.812089:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949221.812101:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000bc928ba2. 00000100:00000001:0.0:1784949221.812110:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949221.812111:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949221.812147:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949221.812165:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949221.812166:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949221.812173:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949221.812174:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.812186:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949221.812191:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee628dd6] to pc [ptlrpcd_00_01+1] req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949221.812193:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949221.812200:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.812208:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949221.812267:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949221.812273:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949221.812280:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949221.812286:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949221.812293:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949221.812301:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008c05042f. 00000020:00000040:1.0:1784949221.812307:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949221.812313:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949221.812317:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949221.812317:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949221.812322:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949221.812325:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1784949221.812325:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949221.812334:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949221.812335:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000e4c24e27. 00000100:00000001:2.0:1784949221.812336:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.812340:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949221.812347:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949221.812351:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.812354:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949221.812359:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949221.812365:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949221.812371:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784949221.812376:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949221.812428:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949221.812430:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949221.812434:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.812436:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949221.812446:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784949221.812450:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008c05042f] to pc [ptlrpcd_00_02+2] req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949221.812453:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949221.812464:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949221.812477:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ee628dd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289682560:0@lo:400: 00000100:00000001:0.0:1784949221.812491:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949221.812501:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949221.812507:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949221.812511:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949221.812518:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 02000000:00000001:0.0:1784949221.812522:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949221.812526:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949221.812531:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007f1947c0. 00000100:00000040:1.0:1784949221.812535:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11498) 00000400:00000010:0.0:1784949221.812543:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000100:00000200:0.0:1784949221.812556:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289682560, portal 25 00000100:00000001:0.0:1784949221.812560:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949221.812563:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637816320 : -115544071735296 : ffff96e9cca5f800) 00000100:00000001:2.0:1784949221.812567:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949221.812575:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.812578:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949221.812585:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949221.812594:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949221.812603:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949221.812604:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:2.0:1784949221.812607:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949221.812614:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00000001:3.0:1784949221.812615:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949221.812626:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289682560, offset 0 00000100:00000001:2.0:1784949221.812642:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784949221.812643:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949221.812649:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949221.812655:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949221.812658:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949221.812662:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949221.812669:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008c05042f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289682688:192.168.202.137@tcp:400: 00000400:00000200:0.0:1784949221.812672:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949221.812683:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949221.812689:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949221.812691:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949221.812695:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784949221.812698:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d280 02000000:00000010:2.0:1784949221.812704:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000098dd29a0. 00000400:00000200:0.0:1784949221.812713:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x88c9 [2] + 0 02000000:00000001:2.0:1784949221.812714:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949221.812726:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a9a70155. 00000400:00000010:2.0:1784949221.812727:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008c614621. 00000400:00000200:0.0:1784949221.812737:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000fba7d3a1 00000400:00000010:2.0:1784949221.812749:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000200:0.0:1784949221.812751:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784949221.812761:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289682688, portal 10 00000400:00000200:0.0:1784949221.812763:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949221.812769:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949221.812774:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637808256 : -115544071743360 : ffff96e9cca5d880) 00000400:00000200:0.0:1784949221.812776:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fba7d3a1 00000400:00000010:0.0:1784949221.812781:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000fba7d3a1. 00000100:00000040:2.0:1784949221.812786:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949221.812789:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949221.812796:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949221.812806:0:337119:0:(events.c:337:request_in_callback()) incoming req@000000001e0f2dc3 x1871651289682560 msgsize 224 00000100:00000001:2.0:1784949221.812813:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949221.812829:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949221.812831:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949221.812838:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000040:0.0:1784949221.812840:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000200:2.0:1784949221.812844:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289682688, offset 0 00000400:00000200:2.0:1784949221.812854:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949221.812867:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949221.812869:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949221.812878:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949221.812885:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949221.812890:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949221.812895:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949221.812901:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3776:3776:235:4294967295] 192.168.202.137@tcp LPNI seq info [2412:2412:8:4294967295] 00000100:00000001:1.0:1784949221.812955:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949221.812961:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289682560 02000000:00000001:1.0:1784949221.812969:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949221.812975:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784949221.812979:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949221.812981:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949221.812992:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949221.813000:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949221.813001:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000100:00100000:1.0:1784949221.813004:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289682560 00000400:00000010:0.0:1784949221.813011:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000020:00000001:1.0:1784949221.813013:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000010:2.0:1784949221.813014:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 00000020:00000040:1.0:1784949221.813021:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949221.813022:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784949221.813030:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:2.0:1784949221.813031:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784949221.813036:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949221.813038:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949221.813050:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:2.0:1784949221.813055:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:1.0:1784949221.813061:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:2.0:1784949221.813068:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949221.813075:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949221.813086:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:0.0:1784949221.813088:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949221.813097:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949221.813099:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.813106:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949221.813107:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000010:1.0:1784949221.813143:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00000100:00000040:2.0:1784949221.813144:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2222 usecs (3 jiffies) 00000100:00000001:0.0:1784949221.813145:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.813157:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1784949221.813159:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000071ebe728. 00000100:00000001:2.0:1784949221.813160:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306161156 00000100:00000001:0.0:1784949221.813169:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949221.813172:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000010:2.0:1784949221.813173:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000082553b41. 00000100:00000001:0.0:1784949221.813177:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306161195 00000100:00000001:2.0:1784949221.813191:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000040:1.0:1784949221.813192:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949221.813201:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949221.813204:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949221.813206:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306161156 00000100:00000001:1.0:1784949221.813207:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.813212:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306161156 00000100:00000010:2.0:1784949221.813221:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f4af3cce. 00000100:00000001:0.0:1784949221.813226:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306161195 00000100:00000001:1.0:1784949221.813227:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949221.813230:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949221.813244:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306161156 00000100:00000001:0.0:1784949221.813248:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306161156 00000100:00000010:2.0:1784949221.813250:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000100:00000001:1.0:1784949221.813252:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949221.813262:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949221.813264:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306161195 00000100:00000001:2.0:1784949221.813278:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306161156 00000100:00000001:1.0:1784949221.813279:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949221.813282:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306161156 00000100:00000001:1.0:1784949221.813290:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000010:2.0:1784949221.813291:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000445487c0. 00000100:00000001:0.0:1784949221.813294:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306161195 00000100:00000001:2.0:1784949221.813303:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00100000:1.0:1784949221.813307:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 804 00000100:00000001:2.0:1784949221.813313:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949221.813314:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949221.813317:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306161156 00000100:00000001:2.0:1784949221.813324:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949221.813326:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741512784 : -115539968038832 : ffff96eac13f6850) 00000100:00000001:2.0:1784949221.813331:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.813332:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949221.813341:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001e0f2dc3 x1871651289682560/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:127/0 lens 224/0 e 0 to 0 dl 1784949232 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.813342:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949221.813343:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949221.813353:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949221.813355:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949221.813356:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949221.813365:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949221.813368:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949221.813370:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000100:00000001:2.0:1784949221.813374:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949221.813381:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949221.813389:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949221.813390:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949221.813391:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949221.813395:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949221.813400:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784949221.813409:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:0.0:1784949221.813411:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949221.813418:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:1.0:1784949221.813419:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000800:00000001:0.0:1784949221.813432:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949221.813435:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001e0f2dc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289682560:12345-0@lo:400: 00000400:00000200:3.0:1784949221.813435:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000800:00000001:0.0:1784949221.813439:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784949221.813447:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000800:00000001:0.0:1784949221.813451:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949221.813452:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289682560 00000100:00000001:3.0:1784949221.813459:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784949221.813462:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949221.813471:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:3.0:1784949221.813478:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949221.813479:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:1.0:1784949221.813479:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949221.813491:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949221.813497:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:0.0:1784949221.813510:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d300 00000020:00000001:1.0:1784949221.813510:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949221.813516:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949221.813525:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949221.813530:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784949221.813532:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:0.0:1784949221.813533:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 175760 00000020:00000001:1.0:1784949221.813541:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949221.813547:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949221.813548:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949221.813550:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949221.813556:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000001:0.0:1784949221.813560:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949221.813563:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:0.0:1784949221.813572:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1784949221.813574:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 02000000:00000001:1.0:1784949221.813585:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949221.813587:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949221.813593:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949221.813594:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949221.813597:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784949221.813604:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949221.813609:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289682560 00010000:00000001:1.0:1784949221.813613:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949221.813620:0:337114:0:(events.c:286:request_in_callback()) Process entered 00010000:00000200:1.0:1784949221.813621:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001e0f2dc3 x1871651289682560/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:127/0 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949221.813626:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949221.813639:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7cbcb5a. 00010000:00000001:1.0:1784949221.813642:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:0.0:1784949221.813646:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000e7cbcb5a x1871651289682688 msgsize 224 00010000:00000001:1.0:1784949221.813647:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949221.813656:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=93 v=5 (1 1 1 1) 00000100:00100000:0.0:1784949221.813664:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949221.813668:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949221.813684:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784949221.813691:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:0.0:1784949221.813695:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949221.813696:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784949221.813705:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000800:00000001:0.0:1784949221.813706:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949221.813711:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949221.813718:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949221.813723:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949221.813730:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000100:00000200:1.0:1784949221.813736:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289682560, offset 224 00000400:00000200:1.0:1784949221.813746:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949221.813752:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949221.813758:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289682688 00000400:00000200:1.0:1784949221.813761:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784949221.813765:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949221.813770:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949221.813773:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949221.813781:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d280 02000000:00000001:2.0:1784949221.813784:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949221.813792:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289682688 00000400:00000200:1.0:1784949221.813793:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x92e5 [1] + 224 00000020:00000001:2.0:1784949221.813800:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949221.813805:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:1.0:1784949221.813807:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949221.813808:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949221.813816:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:1.0:1784949221.813818:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:2.0:1784949221.813826:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949221.813830:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1784949221.813832:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784949221.813840:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.813841:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949221.813849:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:2.0:1784949221.813857:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949221.813865:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000100:00000040:1.0:1784949221.813873:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949221.813876:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000089ec28da. 00000020:00000010:2.0:1784949221.813886:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000059fdf11b. 00000020:00000010:2.0:1784949221.813897:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784949221.813938:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1784949221.813952:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:1.0:1784949221.813953:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:1.0:1784949221.813963:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000100:00000001:2.0:1784949221.813964:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949221.813965:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949221.813972:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949221.813974:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949221.813976:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949221.813977:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949221.813982:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:0.0:1784949221.813985:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784949221.813988:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000001:0.0:1784949221.813991:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784949221.813992:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949221.813994:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000010:0.0:1784949221.813999:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007f1947c0. 00000100:00000001:2.0:1784949221.814002:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949221.814006:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:0.0:1784949221.814009:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00010000:00000001:1.0:1784949221.814012:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:0.0:1784949221.814017:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000020:00000001:1.0:1784949221.814019:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949221.814024:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949221.814026:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949221.814027:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784949221.814039:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001e0f2dc3 x1871651289682560/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:127/0 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949221.814044:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.814048:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949221.814054:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949221.814071:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 780 00000100:00000040:2.0:1784949221.814082:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949221.814092:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554971648 : -115544154579968 : ffff96e9c7b5dc00) 00000100:00100000:1.0:1784949221.814101:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001e0f2dc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289682560:12345-0@lo:400: Request processed in 702us (1285us total) trans 0 rc 0/0 00000100:00000200:0.0:1784949221.814110:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949221.814138:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e7cbcb5a x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:127/0 lens 224/0 e 0 to 0 dl 1784949232 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949221.814152:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949221.814158:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949221.814169:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949221.814175:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949221.814181:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949221.814187:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949221.814192:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 804 02000000:00000001:0.0:1784949221.814197:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949221.814203:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949221.814205:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00001000:0.0:1784949221.814208:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=93 v=5 (5 5 5 5) 00000020:00000010:2.0:1784949221.814213:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:1.0:1784949221.814214:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00001000:0.0:1784949221.814219:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=93 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949221.814223:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784949221.814224:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949221.814228:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:1.0:1784949221.814234:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000071ebe728. 00000100:00000001:0.0:1784949221.814236:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949221.814237:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000020:00000010:1.0:1784949221.814246:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00010000:00000001:0.0:1784949221.814250:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:1.0:1784949221.814256:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000100:00100000:2.0:1784949221.814257:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e7cbcb5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289682688:12345-192.168.202.137@tcp:400: 00010000:00000001:0.0:1784949221.814258:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.814272:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949221.814272:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:2.0:1784949221.814273:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289682688 00000020:00000001:2.0:1784949221.814282:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:1.0:1784949221.814287:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:2.0:1784949221.814288:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:0.0:1784949221.814292:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784949221.814295:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009af7b278. 00000020:00000001:2.0:1784949221.814296:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949221.814308:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:1.0:1784949221.814309:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:2.0:1784949221.814315:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:0.0:1784949221.814318:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949221.814319:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949221.814322:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949221.814330:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949221.814332:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000200:1.0:1784949221.814333:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:2.0:1784949221.814335:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784949221.814339:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949221.814344:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:1.0:1784949221.814347:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f5a5b57b. 00000100:00000040:0.0:1784949221.814349:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949221.814351:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949221.814359:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000449b49f2. 00000020:00000001:2.0:1784949221.814360:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949221.814368:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949221.814375:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:0.0:1784949221.814381:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ee628dd6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289682560:0@lo:400: 02000000:00000010:2.0:1784949221.814384:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000e5e5ec72. 00000100:00000001:0.0:1784949221.814390:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949221.814391:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.814401:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949221.814402:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949221.814409:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949221.814413:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ee628dd6 x1871651289682560/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:2.0:1784949221.814415:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289682688 00010000:00000001:2.0:1784949221.814424:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949221.814433:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e7cbcb5a x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:127/0 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949221.814438:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949221.814445:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949221.814450:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 00010000:00000001:2.0:1784949221.814452:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949221.814459:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949221.814462:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=88 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949221.814463:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1784949221.814469:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949221.814474:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000001:0.0:1784949221.814481:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:2.0:1784949221.814482:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:0.0:1784949221.814487:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 02000000:00000001:2.0:1784949221.814489:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949221.814495:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949221.814497:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:2.0:1784949221.814501:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:0.0:1784949221.814503:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000bc928ba2. 00000100:00000040:2.0:1784949221.814511:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784949221.814514:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:2.0:1784949221.814517:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 02000000:00000001:0.0:1784949221.814519:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949221.814524:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ee628dd6. 00000100:00000200:2.0:1784949221.814525:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289682688, offset 224 00000100:00000001:0.0:1784949221.814530:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949221.814535:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949221.814537:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949221.814545:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949221.814546:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949221.814552:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949221.814556:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949221.814560:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949221.814568:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949221.814580:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3777:3777:235:4294967295] 192.168.202.137@tcp LPNI seq info [2413:2413:8:4294967295] 00000400:00000200:2.0:1784949221.814600:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949221.814610:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949221.814616:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000800:00000200:2.0:1784949221.814625:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949221.814634:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949221.814639:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949221.814657:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949221.814661:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949221.814664:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949221.814666:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949221.814669:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949221.814674:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7cbcb5a x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:127/0 lens 224/224 e 0 to 0 dl 1784949232 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949221.814685:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e7cbcb5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289682688:12345-192.168.202.137@tcp:400: Request processed in 483us (1039us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949221.814695:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 780 00000100:00000040:2.0:1784949221.814698:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949221.814702:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949221.814704:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949221.814709:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000059fdf11b. 00000020:00000010:2.0:1784949221.814714:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949221.814719:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000089ec28da. 00000020:00000040:2.0:1784949221.814726:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949221.814729:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949221.814868:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949221.814878:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000400:00000200:3.0:1784949221.814888:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949221.814900:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949221.814902:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949221.814945:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949221.814949:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949221.814953:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949221.814964:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949221.814967:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000400:00000010:3.0:1784949221.814976:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000400:00000200:1.0:1784949221.814984:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d300 00000100:00000001:3.0:1784949221.814988:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:1.0:1784949221.814997:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x92ed [1] + 224 00000100:00000001:3.0:1784949221.814998:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949221.815010:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949221.815030:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949221.815034:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949221.815041:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949221.815047:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949221.815053:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949221.815066:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949221.815101:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949221.815108:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949221.815142:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949221.815201:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949221.815207:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949221.815214:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949221.815215:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949221.815218:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008c614621. 00000400:00000200:2.0:1784949221.815225:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:2.0:1784949221.815227:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000100:00000001:2.0:1784949221.815231:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949221.815238:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949221.815251:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.815262:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949221.815263:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949221.815266:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949221.815269:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949221.815271:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949221.815273:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949221.815275:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949221.815281:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=88 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949221.815285:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=88 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949221.815288:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949221.815289:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949221.815292:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949221.815296:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949221.815305:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949221.815308:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949221.815310:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949221.815313:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949221.815314:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949221.815317:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.815334:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949221.815337:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949221.815339:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949221.815342:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949221.815351:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008c05042f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289682688:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949221.815356:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949221.815358:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949221.815360:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008c05042f x1871651289682688/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949221.815368:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949221.815370:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949221.815372:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000098dd29a0. 02000000:00000001:2.0:1784949221.815377:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949221.815378:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949221.815380:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949221.815382:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949221.815384:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000e4c24e27. 02000000:00000001:2.0:1784949221.815387:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949221.815389:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949221.815390:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008c05042f. 00000100:00000001:2.0:1784949221.815394:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949221.815395:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949221.815397:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949221.815403:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949221.815404:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949221.815407:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949221.815409:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949221.856002:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306161195, at 4306161196 00000100:00000010:1.0:1784949221.856018:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f4af3cce. 00000100:00000040:3.0:1784949221.856038:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306161195, at 4306161196 00000100:00000010:3.0:1784949221.856066:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000445487c0. 00000100:00000040:2.0:1784949221.856068:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306161195, at 4306161196 00000100:00000040:0.0:1784949221.856072:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306161195, at 4306161196 00000100:00000010:2.0:1784949221.856089:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000010:0.0:1784949221.856095:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000400:00000100:2.0:1784949222.324164:0:344098:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.9:988 00000400:00000100:1.0:1784949222.324169:0:344097:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.7:988 00000400:00000100:0.0:1784949222.324185:0:344094:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.1:988 00000400:00000100:3.0:1784949222.324200:0:344100:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.13:988 00000400:00000100:2.0:1784949222.324291:0:344098:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.9@tcp at host 111.222.142.9:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784949222.324301:0:344097:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.7@tcp at host 111.222.142.7:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784949222.324302:0:344100:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.13@tcp at host 111.222.142.13:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:0.0:1784949222.324304:0:344094:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.1@tcp at host 111.222.142.1:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949222.324320:0:344098:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.9@tcp: down 00000400:00000200:2.0:1784949222.324331:0:344098:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784949222.324334:0:344100:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.13@tcp: down 00000400:00000200:1.0:1784949222.324336:0:344097:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.7@tcp: down 00000400:00000200:0.0:1784949222.324340:0:344094:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.1@tcp: down 00000400:00000200:3.0:1784949222.324351:0:344100:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949222.324362:0:344097:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:2.0:1784949222.324364:0:344095:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.3:988 00000800:00000100:3.0:1784949222.324365:0:344100:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000400:00000200:0.0:1784949222.324375:0:344094:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:3.0:1784949222.324380:0:344100:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000800:00000100:1.0:1784949222.324382:0:344097:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.7@tcp 00000800:00000010:1.0:1784949222.324394:0:344097:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000005678171. 00000400:00000200:3.0:1784949222.324395:0:344100:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:0.0:1784949222.324401:0:344094:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000400:00000200:1.0:1784949222.324411:0:344097:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000010:0.0:1784949222.324412:0:344094:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f495c80e. 00000400:00000100:2.0:1784949222.324415:0:344095:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.3@tcp at host 111.222.142.3:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784949222.324419:0:344100:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949222.324430:0:344094:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949222.324431:0:344097:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949222.324441:0:344100:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11495 health :900 00000400:00000200:2.0:1784949222.324442:0:344095:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.3@tcp: down 00000400:00000200:0.0:1784949222.324456:0:344094:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949222.324458:0:344100:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949222.324460:0:344097:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949222.324466:0:344095:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949222.324476:0:344097:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:3.0:1784949222.324476:0:344100:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:0.0:1784949222.324482:0:344094:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000100:2.0:1784949222.324483:0:344095:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.3@tcp 00000800:00000010:2.0:1784949222.324494:0:344095:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000094ed938f. 00000400:00000200:3.0:1784949222.324497:0:344100:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:0.0:1784949222.324501:0:344094:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:2.0:1784949222.324507:0:344095:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949222.324516:0:344095:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949222.324525:0:344097:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:0.0:1784949222.324539:0:344094:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:2.0:1784949222.324579:0:344095:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949222.324590:0:344095:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000100:3.0:1784949222.324593:0:344099:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.11:988 00000400:00000200:2.0:1784949222.324598:0:344095:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000100:1.0:1784949222.324627:0:344096:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.5:988 00000400:00000200:0.0:1784949222.324655:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000100:3.0:1784949222.324662:0:344099:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.11@tcp at host 111.222.142.11:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949222.324675:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000200:3.0:1784949222.324676:0:344099:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.11@tcp: down 00000400:00000200:3.0:1784949222.324682:0:344099:0:(router.c:1888:lnet_notify()) Old news 00000400:00000001:0.0:1784949222.324694:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000100:3.0:1784949222.324695:0:344099:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.11@tcp 00000400:00000100:1.0:1784949222.324702:0:344096:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.5@tcp at host 111.222.142.5:988 took too long: that node may be hung or experiencing high load. 00000400:00000001:0.0:1784949222.324703:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000010:3.0:1784949222.324704:0:344099:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ca1124f5. 00000400:00000200:0.0:1784949222.324715:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000200:3.0:1784949222.324717:0:344099:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949222.324721:0:344096:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.5@tcp: down 00000400:00000010:0.0:1784949222.324726:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000400:00000200:1.0:1784949222.324730:0:344096:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784949222.324738:0:344099:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949222.324744:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949222.324748:0:344099:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000100:1.0:1784949222.324749:0:344096:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000800:00000010:1.0:1784949222.324758:0:344096:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000076916763. 00000400:00000200:0.0:1784949222.324762:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:3.0:1784949222.324762:0:344099:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:1.0:1784949222.324767:0:344096:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:0.0:1784949222.324780:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000043c3e1d8. 00000400:00000200:1.0:1784949222.324780:0:344096:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949222.324799:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:3.0:1784949222.324800:0:344099:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:0.0:1784949222.324816:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:1.0:1784949222.324824:0:344096:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949222.324831:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:1.0:1784949222.324837:0:344096:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:1.0:1784949222.324846:0:344096:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:0.0:1784949222.324859:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000400:00000001:0.0:1784949222.324867:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949222.324870:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949222.324873:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000010:0.0:1784949222.324877:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000400:00000200:0.0:1784949222.324883:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949222.324889:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:0.0:1784949222.324894:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000684a4a3c. 00000400:00000200:0.0:1784949222.324903:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-110 00000400:00000200:0.0:1784949222.324970:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -110 00000400:00000200:0.0:1784949222.324977:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:0.0:1784949222.324986:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:0.0:1784949222.324992:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949222.324996:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949222.324999:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000010:0.0:1784949222.325004:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000400:00000200:0.0:1784949222.325009:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949222.325015:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:0.0:1784949222.325019:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001df3b40e. 00000400:00000200:0.0:1784949222.325026:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000200:0.0:1784949222.325030:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:0.0:1784949222.325035:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:0.0:1784949222.325040:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 00000400:00000001:0.0:1784949222.325044:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949222.325046:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949222.325049:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:0.0:1784949222.325052:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000400:00000200:0.0:1784949222.325055:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949222.325060:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:0.0:1784949222.325064:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f1904ca. 00000400:00000200:0.0:1784949222.325070:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-110 00000400:00000200:0.0:1784949222.325074:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -110 00000400:00000200:0.0:1784949222.325080:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00000400:00000200:0.0:1784949222.325085:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:0.0:1784949222.325089:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949222.325091:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949222.325094:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:0.0:1784949222.325097:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000400:00000200:0.0:1784949222.325101:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949222.325105:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:0.0:1784949222.325138:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009802f36f. 00000400:00000200:0.0:1784949222.325148:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:0.0:1784949222.325152:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:0.0:1784949222.325158:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000400:00000200:0.0:1784949222.325164:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:0.0:1784949222.325168:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949222.325170:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949222.325173:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:0.0:1784949222.325175:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:0.0:1784949222.325180:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:0.0:1784949222.325184:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:0.0:1784949222.325189:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000072ce2c8. 00000400:00000200:0.0:1784949222.325195:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:0.0:1784949222.325199:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:0.0:1784949222.325205:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00010000:00000001:2.0:1784949222.644026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949222.644035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949222.644042:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949222.644044:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949222.644183:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949222.644189:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949222.644193:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949222.836050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949222.836067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949222.836070:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949222.836076:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949222.836080:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949222.836085:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949222.836088:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949223.028020:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949223.028032:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000f2ac7018. 00000400:00000010:3.0:1784949223.028043:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000035b54a67. 00000400:00000010:3.0:1784949223.028050:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000467f058f. 00000400:00000010:3.0:1784949223.028060:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b5ecaa25. 00000400:00000200:3.0:1784949223.028086:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949223.028155:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949223.028164:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949223.028169:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3778:3778:241:4294967295] 192.168.202.137@tcp LPNI seq info [2414:2414:8:4294967295] 00000400:00000200:3.0:1784949223.028178:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949223.028185:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949223.028193:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949223.028199:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003547cd5f. 00000800:00000200:3.0:1784949223.028207:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949223.028216:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949223.028221:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949223.028244:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11497 health :400 00000800:00000200:0.0:1784949223.028387:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949223.028393:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003547cd5f. 00000400:00000200:0.0:1784949223.028403:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949223.028411:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:3.0:1784949223.028412:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949223.028417:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949223.028422:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949223.028428:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949223.028433:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949223.028440:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949223.028450:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949223.028465:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949223.028476:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949223.028486:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949223.028492:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3779:3779:241:4294967295] 192.168.202.137@tcp LPNI seq info [2415:2415:8:4294967295] 00000400:00000200:3.0:1784949223.028503:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949223.028509:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949223.028513:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949223.028519:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949223.028525:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949223.028528:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949223.028611:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949223.028632:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949223.028637:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000800:00000001:1.0:1784949223.028641:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949223.028643:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949223.028646:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949223.028653:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949223.028655:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949223.028659:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949223.028662:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949223.028670:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784949223.028681:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9301 00000800:00000001:1.0:1784949223.028686:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949223.028698:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949223.028700:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949223.028702:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949223.028707:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000400:00000010:1.0:1784949223.028709:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000400:00000200:1.0:1784949223.028712:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949223.028714:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000f2ac7018. 00000400:00000010:1.0:1784949223.028717:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000035b54a67. 00000800:00000001:1.0:1784949223.028723:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949223.028724:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949223.668032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949223.668041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949223.668048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949223.668050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949223.668195:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949223.668203:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949223.668209:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949223.860039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949223.860058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949223.860060:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949223.860068:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949223.860072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949223.860077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949223.860082:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949224.692041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949224.692050:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949224.692060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949224.692062:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949224.692193:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949224.692201:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949224.692206:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949224.884068:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949224.884085:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949224.884108:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949224.884115:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949224.884119:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949224.884123:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949224.884127:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949225.076077:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949225.076147:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000b5a9b7be. 00000400:00000010:3.0:1784949225.076163:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009f047fe3. 00000400:00000010:3.0:1784949225.076170:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000400:00000010:3.0:1784949225.076180:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e07f0bcd. 00000400:00000200:3.0:1784949225.076186:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949225.076205:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949225.076214:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949225.076220:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3780:3780:241:4294967295] 192.168.202.137@tcp LPNI seq info [2416:2416:8:4294967295] 00000400:00000200:3.0:1784949225.076229:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949225.076239:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949225.076245:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949225.076252:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949225.076260:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949225.076271:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949225.076277:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949225.076305:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11499 health :900 00000800:00000200:0.0:1784949225.076492:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949225.076501:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:0.0:1784949225.076513:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949225.076514:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949225.076524:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:3.0:1784949225.076532:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949225.076533:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:3.0:1784949225.076544:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949225.076545:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949225.076556:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949225.076570:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949225.076587:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949225.076599:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949225.076611:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949225.076616:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3781:3781:241:4294967295] 192.168.202.137@tcp LPNI seq info [2417:2417:8:4294967295] 00000400:00000200:3.0:1784949225.076627:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949225.076633:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949225.076637:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949225.076643:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949225.076648:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949225.076651:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 3, nob 144 niov 1 nkiov 1 00000800:00000200:3.0:1784949225.076778:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784949225.076779:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949225.076784:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000800:00000001:1.0:1784949225.076792:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949225.076794:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949225.076800:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949225.076811:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949225.076812:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949225.076821:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949225.076828:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949225.076840:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784949225.076859:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9305 00000800:00000001:1.0:1784949225.076867:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949225.076887:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949225.076891:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949225.076895:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949225.076902:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000400:00000010:1.0:1784949225.076937:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000400:00000200:1.0:1784949225.076943:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949225.076947:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000b5a9b7be. 00000400:00000010:1.0:1784949225.076952:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009f047fe3. 00000800:00000001:1.0:1784949225.076960:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949225.076964:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949225.716037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949225.716046:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949225.716053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949225.716055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949225.716184:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949225.716190:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949225.716194:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949225.908003:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949225.908021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949225.908024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949225.908031:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949225.908035:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949225.908039:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949225.908044:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949226.740034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949226.740044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949226.740052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949226.740054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949226.740191:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949226.740198:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949226.740203:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949226.932026:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949226.932026:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949226.932038:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949226.932039:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949226.932042:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949226.932045:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784949226.932049:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb4f6eba. 00000100:00000001:1.0:1784949226.932049:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949226.932055:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949226.932059:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949226.932061:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949226.932086:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949226.932092:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949226.932098:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949226.932109:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00000001:0.0:1784949226.932119:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949226.932131:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949226.932147:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949226.932151:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949226.932160:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb4f6eba] to pc [ptlrpcd_00_03+3] req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949226.932196:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949226.932200:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949226.932205:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949226.932209:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949226.932213:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949226.932215:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:3.0:1784949226.932220:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1784949226.932222:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b16a92e. 00000100:00000001:3.0:1784949226.932225:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1784949226.932229:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949226.932235:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:3.0:1784949226.932235:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949226.932240:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949226.932244:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949226.932254:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d104522b. 00000100:00000001:3.0:1784949226.932258:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949226.932261:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949226.932263:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932265:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949226.932270:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949226.932271:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949226.932274:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1784949226.932278:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949226.932286:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949226.932295:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289682816:0@lo:400: 00000100:00000001:0.0:1784949226.932296:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949226.932300:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949226.932305:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949226.932310:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b16a92e] to pc [ptlrpcd_00_00+0] req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949226.932316:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949226.932340:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949226.932349:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949226.932359:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dd0652fb. 02000000:00000001:3.0:1784949226.932364:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949226.932374:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000400:00000010:3.0:1784949226.932387:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000100:00000001:0.0:1784949226.932388:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932398:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1784949226.932402:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289682816, portal 25 00000100:00000040:0.0:1784949226.932403:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11503) 00000100:00000001:2.0:1784949226.932405:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949226.932411:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949226.932413:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949226.932419:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549148160 : -115544160403456 : ffff96e9c75d0000) 00000100:00000040:2.0:1784949226.932426:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949226.932432:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949226.932444:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949226.932454:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932477:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949226.932479:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949226.932481:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949226.932488:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784949226.932489:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:3.0:1784949226.932493:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 02000000:00000001:2.0:1784949226.932494:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949226.932504:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949226.932505:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289682816, offset 0 00000100:00100000:2.0:1784949226.932512:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289682944:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784949226.932522:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949226.932524:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949226.932531:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949226.932533:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949226.932536:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949226.932542:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000089ec28da. 00000400:00000200:3.0:1784949226.932551:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784949226.932552:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949226.932565:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008c614621. 00000400:00000010:2.0:1784949226.932575:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000200:3.0:1784949226.932581:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d380 00000100:00000200:2.0:1784949226.932585:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289682944, portal 10 00000100:00000001:2.0:1784949226.932592:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949226.932594:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482521600 : -115544227030016 : ffff96e9c3645c00) 00000400:00000200:3.0:1784949226.932598:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x88ed [2] + 0 00000400:00000010:3.0:1784949226.932606:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000af4b2538. 00000100:00000040:2.0:1784949226.932610:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949226.932617:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000034386f9e 00000100:00000001:2.0:1784949226.932628:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949226.932633:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949226.932634:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949226.932642:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000200:3.0:1784949226.932649:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:2.0:1784949226.932652:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289682944, offset 0 00000400:00000200:3.0:1784949226.932674:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000034386f9e 00000400:00000010:3.0:1784949226.932678:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000034386f9e. 00000100:00000001:3.0:1784949226.932683:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:2.0:1784949226.932687:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000200:3.0:1784949226.932688:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949226.932693:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000000c7e43f4 x1871651289682816 msgsize 224 00000100:00100000:3.0:1784949226.932698:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:2.0:1784949226.932701:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:3.0:1784949226.932702:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949226.932709:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949226.932714:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949226.932718:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784949226.932721:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949226.932722:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949226.932727:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3782:3782:241:4294967295] 192.168.202.137@tcp LPNI seq info [2418:2418:8:4294967295] 00000400:00000200:2.0:1784949226.932737:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949226.932742:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949226.932746:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000800:00000010:2.0:1784949226.932748:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000400:00000010:3.0:1784949226.932751:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000100:00000001:3.0:1784949226.932756:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:2.0:1784949226.932760:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:3.0:1784949226.932764:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949226.932772:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000200:2.0:1784949226.932774:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:0.0:1784949226.932780:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289682816 00000800:00000200:2.0:1784949226.932781:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949226.932782:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949226.932787:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:3.0:1784949226.932790:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949226.932792:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949226.932798:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949226.932801:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932803:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949226.932805:0:337122:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1784949226.932806:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949226.932810:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2225 usecs (3 jiffies) 00000100:00100000:0.0:1784949226.932813:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289682816 00000100:00000001:3.0:1784949226.932813:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932816:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306166275 00000100:00000001:3.0:1784949226.932821:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306166314 00000020:00000001:0.0:1784949226.932822:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000010:2.0:1784949226.932823:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000020:00000040:0.0:1784949226.932828:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949226.932832:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949226.932834:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000040:0.0:1784949226.932837:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949226.932841:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306166275 00000020:00000001:0.0:1784949226.932843:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:2.0:1784949226.932846:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008a834f68. 00000100:00000001:3.0:1784949226.932847:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306166275 00000020:00000001:0.0:1784949226.932850:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949226.932854:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949226.932857:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306166314 00000100:00000001:0.0:1784949226.932858:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949226.932861:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306166275 00000100:00000001:3.0:1784949226.932865:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306166275 00000020:00000010:0.0:1784949226.932867:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000010:2.0:1784949226.932867:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e2039f57. 00000100:00000001:3.0:1784949226.932871:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306166314 00000100:00000001:2.0:1784949226.932874:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949226.932876:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949226.932880:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306166275 00000020:00000010:0.0:1784949226.932883:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00000001:3.0:1784949226.932883:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306166275 00000100:00000010:2.0:1784949226.932884:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a3d0cc2f. 00000100:00000001:2.0:1784949226.932893:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784949226.932893:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306166314 00000020:00000010:0.0:1784949226.932899:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:2.0:1784949226.932899:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949226.932901:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306166275 00000100:00000001:3.0:1784949226.932937:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949226.932942:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932943:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949226.932945:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000005f71537f. 00000100:00000001:3.0:1784949226.932950:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932951:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949226.932955:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949226.932958:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949226.932959:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949226.932968:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949226.932969:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949226.932972:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949226.932973:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949226.932974:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949226.932981:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949226.932982:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949226.932983:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949226.932987:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949226.932988:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949226.932989:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949226.932993:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949226.932994:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949226.932995:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.932999:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949226.933002:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949226.933004:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949226.933009:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000100:00000001:0.0:1784949226.933011:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949226.933016:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949226.933023:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949226.933027:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000100:00000001:0.0:1784949226.933030:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949226.933030:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000100:00000001:1.0:1784949226.933036:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949226.933044:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949226.933044:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949226.933053:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949226.933055:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:3.0:1784949226.933057:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949226.933058:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:1.0:1784949226.933061:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949226.933064:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949226.933069:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949226.933087:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 805 00000400:00000200:3.0:1784949226.933098:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:0.0:1784949226.933099:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949226.933107:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949226.933108:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741514832 : -115539968036784 : ffff96eac13f7050) 00000100:00000001:1.0:1784949226.933112:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784949226.933117:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d400 00000100:00000040:0.0:1784949226.933122:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000c7e43f4 x1871651289682816/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:132/0 lens 224/0 e 0 to 0 dl 1784949237 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949226.933128:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 175984 00000800:00000001:3.0:1784949226.933140:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949226.933147:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:3.0:1784949226.933149:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949226.933154:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000400:00000200:3.0:1784949226.933154:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949226.933162:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949226.933162:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:0.0:1784949226.933168:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00000001:3.0:1784949226.933168:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949226.933172:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:3.0:1784949226.933177:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000074c559e1. 00000100:00100000:0.0:1784949226.933180:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000c7e43f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289682816:12345-0@lo:400: 00000100:00000040:3.0:1784949226.933182:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000074c559e1 x1871651289682944 msgsize 224 00000100:00000200:0.0:1784949226.933194:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289682816 00000100:00100000:3.0:1784949226.933194:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:0.0:1784949226.933201:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949226.933205:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:3.0:1784949226.933206:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:0.0:1784949226.933209:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949226.933214:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949226.933215:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:3.0:1784949226.933219:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949226.933220:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949226.933228:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949226.933231:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949226.933235:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949226.933238:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949226.933242:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949226.933245:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949226.933249:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949226.933252:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:0.0:1784949226.933253:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:1.0:1784949226.933256:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289682944 02000000:00000001:1.0:1784949226.933260:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000010:0.0:1784949226.933261:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000035bbf39d. 00000100:00000001:1.0:1784949226.933262:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949226.933265:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949226.933269:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949226.933270:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949226.933279:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949226.933279:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289682944 00000020:00000001:1.0:1784949226.933284:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:0.0:1784949226.933286:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949226.933287:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949226.933291:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00010000:00000040:0.0:1784949226.933295:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289682816 00000020:00000040:1.0:1784949226.933295:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949226.933300:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:0.0:1784949226.933305:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:1.0:1784949226.933305:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949226.933311:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949226.933315:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00010000:00000200:0.0:1784949226.933316:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000c7e43f4 x1871651289682816/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:132/0 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949226.933320:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949226.933325:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000020:00000010:1.0:1784949226.933335:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00010000:00000001:0.0:1784949226.933338:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:1.0:1784949226.933340:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000071ebe728. 00010000:00000001:0.0:1784949226.933343:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949226.933346:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00001000:0.0:1784949226.933352:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=98 v=5 (1 1 1 1) 00000100:00000040:1.0:1784949226.933354:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949226.933359:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949226.933362:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949226.933364:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949226.933365:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949226.933373:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949226.933374:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949226.933377:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949226.933381:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784949226.933381:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949226.933389:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949226.933392:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949226.933397:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949226.933398:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949226.933404:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949226.933410:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000100:00000001:1.0:1784949226.933411:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949226.933415:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:0.0:1784949226.933418:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289682816, offset 224 00000100:00100000:1.0:1784949226.933423:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 781 00000100:00000040:1.0:1784949226.933428:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949226.933432:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554956160 : -115544154595456 : ffff96e9c7b59f80) 00000400:00000200:0.0:1784949226.933434:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:1.0:1784949226.933439:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000074c559e1 x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:132/0 lens 224/0 e 0 to 0 dl 1784949237 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949226.933451:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949226.933454:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949226.933459:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784949226.933461:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949226.933466:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000400:00000200:0.0:1784949226.933467:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d380 00000100:00100000:1.0:1784949226.933475:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000074c559e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289682944:12345-192.168.202.137@tcp:400: 00000400:00000200:0.0:1784949226.933482:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9309 [1] + 224 00000100:00000200:1.0:1784949226.933484:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289682944 00000020:00000001:1.0:1784949226.933488:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949226.933491:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949226.933494:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949226.933496:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:0.0:1784949226.933497:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784949226.933498:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949226.933506:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:0.0:1784949226.933511:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:1.0:1784949226.933511:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949226.933534:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949226.933538:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784949226.933540:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1784949226.933542:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949226.933549:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949226.933555:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000200:0.0:1784949226.933557:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949226.933559:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949226.933566:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009af7b278. 02000000:00000001:1.0:1784949226.933573:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949226.933580:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949226.933587:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949226.933594:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289682944 00010000:00000001:1.0:1784949226.933606:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:0.0:1784949226.933612:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784949226.933614:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000074c559e1 x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:132/0 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949226.933656:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949226.933661:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949226.933685:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=93 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949226.933689:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949226.933692:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949226.933694:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784949226.933696:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1784949226.933697:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949226.933701:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949226.933704:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949226.933707:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949226.933709:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:1.0:1784949226.933711:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000400:00000010:0.0:1784949226.933715:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000100:00000001:0.0:1784949226.933722:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949226.933724:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949226.933728:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949226.933732:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000200:1.0:1784949226.933736:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289682944, offset 224 00010000:00000001:0.0:1784949226.933737:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949226.933742:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949226.933748:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949226.933749:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949226.933759:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c7e43f4 x1871651289682816/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:132/0 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949226.933759:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949226.933773:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3783:3783:241:4294967295] 192.168.202.137@tcp LPNI seq info [2419:2419:8:4294967295] 00000100:00000001:3.0:1784949226.933774:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949226.933779:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949226.933788:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1784949226.933793:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000c7e43f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289682816:12345-0@lo:400: Request processed in 640us (1093us total) trans 0 rc 0/0 00000400:00000001:3.0:1784949226.933793:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949226.933798:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000400:00000200:1.0:1784949226.933801:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000400:00000200:3.0:1784949226.933808:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000400:00000010:3.0:1784949226.933814:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000800:00000200:1.0:1784949226.933820:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949226.933821:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784949226.933822:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 805 00000800:00000010:1.0:1784949226.933829:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b80ce7eb. 00000100:00000200:3.0:1784949226.933830:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949226.933831:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949226.933838:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000200:1.0:1784949226.933841:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:3.0:1784949226.933854:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949226.933857:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949226.933864:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b80ce7eb type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949226.933877:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949226.933877:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949226.933881:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949226.933882:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949226.933886:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000005f71537f. 00000100:00000001:3.0:1784949226.933887:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:1.0:1784949226.933890:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 02000000:00000001:3.0:1784949226.933892:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949226.933896:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:0.0:1784949226.933897:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00010000:00000001:1.0:1784949226.933898:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949226.933900:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949226.933902:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949226.933933:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 02000000:00000001:3.0:1784949226.933933:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949226.933939:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000100:00001000:3.0:1784949226.933941:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=98 v=5 (5 5 5 5) 00000100:00000040:1.0:1784949226.933942:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000074c559e1 x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:132/0 lens 224/224 e 0 to 0 dl 1784949237 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:3.0:1784949226.933951:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=98 v=5 (1 1 1 1) 00000020:00000040:0.0:1784949226.933953:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949226.933960:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949226.933964:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949226.933965:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:3.0:1784949226.933968:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00100000:1.0:1784949226.933971:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000074c559e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289682944:12345-192.168.202.137@tcp:400: Request processed in 510us (782us total) trans 0 rc 0/0 02000000:00000010:0.0:1784949226.933972:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000035bbf39d. 00010000:00000001:3.0:1784949226.933972:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949226.933980:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949226.933983:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949226.933990:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949226.933990:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949226.933995:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 781 00000100:00000200:0.0:1784949226.934001:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:1.0:1784949226.934002:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949226.934010:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:0.0:1784949226.934013:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000007f1947c0. 00000100:00000001:3.0:1784949226.934014:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949226.934015:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949226.934021:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000071ebe728. 00000100:00000001:3.0:1784949226.934022:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:0.0:1784949226.934024:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003c50331a. 00000800:00000200:2.0:1784949226.934026:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949226.934026:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949226.934028:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000800:00000010:2.0:1784949226.934034:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b80ce7eb. 00000020:00000010:1.0:1784949226.934036:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000100:00000040:3.0:1784949226.934037:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949226.934042:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1784949226.934044:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949226.934051:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949226.934055:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949226.934064:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000100:00100000:3.0:1784949226.934065:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289682816:0@lo:400: 00000800:00000001:0.0:1784949226.934067:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784949226.934068:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000100:00000001:2.0:1784949226.934093:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949226.934093:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949226.934097:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949226.934098:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:0.0:1784949226.934100:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949226.934105:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb4f6eba x1871651289682816/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949226.934107:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949226.934113:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949226.934122:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:0.0:1784949226.934128:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:3.0:1784949226.934129:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949226.934135:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dd0652fb. 02000000:00000001:3.0:1784949226.934142:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:0.0:1784949226.934143:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d400 00000020:00000001:3.0:1784949226.934146:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949226.934151:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:0.0:1784949226.934158:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9311 [1] + 224 00000020:00000001:3.0:1784949226.934158:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949226.934163:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 00000800:00000001:0.0:1784949226.934169:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949226.934171:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949226.934176:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949226.934178:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb4f6eba. 00000800:00000001:0.0:1784949226.934180:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949226.934184:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000400:00000200:0.0:1784949226.934186:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949226.934188:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949226.934192:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949226.934198:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949226.934199:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949226.934203:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949226.934205:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949226.934205:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949226.934211:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949226.934215:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949226.934229:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949226.934252:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949226.934260:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949226.934264:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949226.934284:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949226.934287:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949226.934291:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949226.934292:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949226.934294:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008c614621. 00000400:00000200:2.0:1784949226.934298:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:2.0:1784949226.934299:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000100:00000001:2.0:1784949226.934301:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949226.934305:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949226.934312:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949226.934318:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949226.934319:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949226.934320:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949226.934322:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949226.934323:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949226.934324:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949226.934326:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949226.934330:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=93 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949226.934332:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=93 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949226.934334:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949226.934335:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949226.934337:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949226.934340:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949226.934351:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.934354:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949226.934355:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949226.934357:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949226.934358:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949226.934360:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949226.934365:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949226.934366:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949226.934367:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949226.934369:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949226.934374:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289682944:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949226.934377:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949226.934379:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949226.934381:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b16a92e x1871651289682944/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949242 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949226.934386:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949226.934387:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949226.934389:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000089ec28da. 02000000:00000001:2.0:1784949226.934391:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949226.934392:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949226.934393:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949226.934395:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949226.934396:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d104522b. 02000000:00000001:2.0:1784949226.934398:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949226.934399:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949226.934401:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b16a92e. 00000100:00000001:2.0:1784949226.934403:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949226.934404:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949226.934406:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949226.934409:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949226.934410:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949226.934412:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949226.934413:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949226.975033:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306166314, at 4306166315 00000100:00000040:0.0:1784949226.975049:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306166314, at 4306166315 00000100:00000040:2.0:1784949226.975050:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306166314, at 4306166315 00000100:00000010:1.0:1784949226.975052:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008a834f68. 00000100:00000010:0.0:1784949226.975068:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000010:2.0:1784949226.975097:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00000100:00000040:3.0:1784949226.976013:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306166314, at 4306166316 00000100:00000010:3.0:1784949226.976025:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a3d0cc2f. 00010000:00000001:2.0:1784949227.764056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949227.764095:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949227.764106:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949227.764108:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949227.764200:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949227.764207:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949227.764211:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949227.956131:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949227.956149:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949227.956152:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949227.956159:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949227.956163:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949227.956168:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949227.956173:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949228.788037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949228.788046:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949228.788076:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949228.788078:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949228.788195:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949228.788202:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949228.788207:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949228.980115:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949228.980133:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949228.980136:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949228.980142:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949228.980145:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949228.980150:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949228.980154:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949229.812086:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949229.812096:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949229.812104:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949229.812106:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949229.812227:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949229.812235:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949229.812239:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949230.004039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949230.004097:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949230.004100:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949230.004108:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949230.004112:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949230.004118:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949230.004122:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1784949230.516093:0:344012:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.15:988 00000400:00000100:0.0:1784949230.516130:0:343956:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.28:988 00000400:00000100:2.0:1784949230.516168:0:344012:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.15@tcp at host 111.222.142.15:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949230.516186:0:344012:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.15@tcp: down 00000400:00000200:2.0:1784949230.516195:0:344012:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:0.0:1784949230.516207:0:343956:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.28@tcp at host 111.222.142.28:988 took too long: that node may be hung or experiencing high load. 00000800:00000100:2.0:1784949230.516210:0:344012:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000800:00000010:2.0:1784949230.516221:0:344012:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:0.0:1784949230.516233:0:343956:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.28@tcp: down 00000400:00000200:2.0:1784949230.516234:0:344012:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949230.516247:0:344012:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000800:00000200:0.0:1784949230.516255:0:343956:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.142.28@tcp down 00000400:00000200:2.0:1784949230.516257:0:344012:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11503 health :900 00000400:00000200:2.0:1784949230.516269:0:344012:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949230.516276:0:344012:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:2.0:1784949230.516285:0:344012:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:3.0:1784949230.516346:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949230.516359:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:3.0:1784949230.516368:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949230.516370:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949230.516375:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:3.0:1784949230.516378:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000400:00000200:3.0:1784949230.516384:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949230.516389:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:3.0:1784949230.516394:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000033c25eed. 00000400:00000200:3.0:1784949230.516401:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:3.0:1784949230.516404:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:3.0:1784949230.516409:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784949230.836114:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949230.836123:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949230.836130:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949230.836131:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949230.836212:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949230.836217:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949230.836220:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949231.028022:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949231.028082:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949231.028085:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949231.028091:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949231.028095:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949231.028099:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949231.028103:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949231.220131:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949231.220144:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000003ab84a81. 00000400:00000010:3.0:1784949231.220159:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dd0652fb. 00000400:00000010:3.0:1784949231.220170:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000400:00000010:3.0:1784949231.220184:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000061508776. 00000400:00000200:3.0:1784949231.220193:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949231.220216:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949231.220228:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949231.220235:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3784:3784:242:4294967295] 192.168.202.137@tcp LPNI seq info [2420:2420:8:4294967295] 00000400:00000200:3.0:1784949231.220247:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949231.220260:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949231.220271:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949231.220280:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000800:00000200:3.0:1784949231.220292:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949231.220306:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949231.220314:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949231.220350:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11505 health :900 00000800:00000200:0.0:1784949231.220596:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949231.220607:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:0.0:1784949231.220620:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949231.220635:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949231.220647:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949231.220654:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949231.220659:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949231.220681:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949231.220688:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949231.220697:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949231.220712:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949231.220736:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949231.220752:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949231.220768:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949231.220776:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3785:3785:242:4294967295] 192.168.202.137@tcp LPNI seq info [2421:2421:8:4294967295] 00000400:00000200:2.0:1784949231.220793:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949231.220801:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949231.220809:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000adf08f9b. 00000800:00000200:2.0:1784949231.220818:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949231.220827:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949231.220833:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949231.220956:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949231.221033:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949231.221075:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949231.221081:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949231.221087:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949231.221093:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1784949231.221104:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000010:2.0:1784949231.221105:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000adf08f9b. 00000400:00000200:2.0:1784949231.221119:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949231.221137:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949231.221141:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949231.221144:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9325 00000800:00000001:0.0:1784949231.221163:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949231.221181:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949231.221187:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949231.221197:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949231.221207:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000400:00000010:0.0:1784949231.221213:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000400:00000200:0.0:1784949231.221218:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949231.221223:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000003ab84a81. 00000400:00000010:0.0:1784949231.221230:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dd0652fb. 00000800:00000001:0.0:1784949231.221241:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949231.221246:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949231.860024:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949231.860057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949231.860066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949231.860071:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949231.860074:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949231.860078:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949231.860082:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949231.860086:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949231.860088:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949231.860091:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949231.860227:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949231.860234:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949231.860238:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949232.052026:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949232.052060:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949232.052074:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b16a92e. 00000020:00000040:0.0:1784949232.052083:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949232.052090:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949232.052095:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949232.052104:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949232.052116:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000635133da. 00000100:00000001:0.0:1784949232.052128:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949232.052135:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1784949232.052142:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949232.052156:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949232.052159:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949232.052164:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949232.052165:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949232.052179:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949232.052184:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b16a92e] to pc [ptlrpcd_00_01+1] req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949232.052188:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949232.052200:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949232.052209:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.052227:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949232.052232:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949232.052238:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:2.0:1784949232.052238:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949232.052249:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1784949232.052250:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1784949232.052256:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949232.052261:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949232.052273:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949232.052276:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949232.052281:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949232.052284:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949232.052288:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:0.0:1784949232.052297:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a3e309c2. 00000100:00000040:2.0:1784949232.052300:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784949232.052304:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949232.052311:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949232.052316:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949232.052321:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949232.052323:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949232.052333:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949232.052335:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000057c7ab4. 00000100:00000001:2.0:1784949232.052337:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949232.052347:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.052348:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949232.052364:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949232.052371:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1784949232.052375:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949232.052388:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.052392:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949232.052396:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949232.052402:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289683072:0@lo:400: 00000100:00000040:0.0:1784949232.052410:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e309c2] to pc [ptlrpcd_00_02+2] req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949232.052420:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949232.052436:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949232.052440:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.052446:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949232.052447:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784949232.052457:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11509) 02000000:00000010:2.0:1784949232.052461:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000089ec28da. 00000100:00000001:3.0:1784949232.052461:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949232.052472:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949232.052475:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:3.0:1784949232.052483:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949232.052486:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008c614621. 00000100:00000001:3.0:1784949232.052495:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:2.0:1784949232.052504:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000100:00000200:2.0:1784949232.052519:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289683072, portal 25 00000100:00000001:2.0:1784949232.052526:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949232.052531:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482521600 : -115544227030016 : ffff96e9c3645c00) 00000100:00000040:2.0:1784949232.052543:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949232.052560:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949232.052567:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949232.052573:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00000200:2.0:1784949232.052579:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289683072, offset 0 00000400:00000200:2.0:1784949232.052593:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949232.052619:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949232.052637:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949232.052645:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d480 00000100:00000001:3.0:1784949232.052649:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949232.052657:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949232.052664:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949232.052671:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8915 [2] + 0 00000100:00000040:3.0:1784949232.052679:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949232.052685:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dc4daa2d. 00000100:00000001:1.0:1784949232.052687:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949232.052696:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949232.052702:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000007790144d 00000100:00000001:3.0:1784949232.052718:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1784949232.052722:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949232.052722:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949232.052734:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784949232.052742:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007790144d 00000400:00000010:2.0:1784949232.052747:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000007790144d. 00000100:00000001:2.0:1784949232.052755:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949232.052759:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949232.052765:0:337119:0:(events.c:337:request_in_callback()) incoming req@0000000093c39287 x1871651289683072 msgsize 224 00000100:00100000:2.0:1784949232.052774:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949232.052778:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949232.052804:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949232.052816:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:2.0:1784949232.052821:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000100:00000001:2.0:1784949232.052827:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949232.052835:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949232.052856:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949232.052863:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:2.0:1784949232.052864:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949232.052872:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289683072 02000000:00000001:1.0:1784949232.052882:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949232.052890:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949232.052894:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949232.052900:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949232.052902:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949232.052986:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949232.052989:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949232.053002:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289683072 00000100:00000001:2.0:1784949232.053003:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306171435 00000020:00000001:1.0:1784949232.053008:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949232.053014:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:2.0:1784949232.053017:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e2039f57. 00000020:00000001:1.0:1784949232.053018:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949232.053024:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949232.053054:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:3.0:1784949232.053056:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949232.053064:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949232.053066:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949232.053066:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1784949232.053077:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949232.053083:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949232.053090:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306171435 00000100:00100000:3.0:1784949232.053090:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a3e309c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289683200:192.168.202.137@tcp:400: 00000020:00000010:1.0:1784949232.053093:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784949232.053106:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949232.053107:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1784949232.053108:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000010de4e21. 00000020:00000010:1.0:1784949232.053116:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 02000000:00000001:3.0:1784949232.053118:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949232.053122:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1784949232.053127:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949232.053132:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:1.0:1784949232.053134:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00000100:00000001:2.0:1784949232.053136:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306171435 02000000:00000010:3.0:1784949232.053138:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000072ce5ef0. 00000020:00000010:1.0:1784949232.053147:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000071ebe728. 02000000:00000001:3.0:1784949232.053150:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949232.053154:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000020:00000010:1.0:1784949232.053160:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000400:00000010:3.0:1784949232.053163:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000100:00000001:2.0:1784949232.053166:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000400:00000010:3.0:1784949232.053177:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 00000100:00000040:1.0:1784949232.053182:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949232.053183:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306171435 00000100:00000200:3.0:1784949232.053186:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289683200, portal 10 00000100:00000001:1.0:1784949232.053193:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949232.053197:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949232.053201:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000010:2.0:1784949232.053201:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a2179890. 00000100:00000001:3.0:1784949232.053203:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469567232 : -115544239984384 : ffff96e9c29eb100) 00000100:00000001:1.0:1784949232.053211:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949232.053216:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000040:3.0:1784949232.053222:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949232.053228:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949232.053230:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949232.053244:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949232.053248:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949232.053256:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949232.053257:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949232.053261:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949232.053266:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000100:00000001:2.0:1784949232.053270:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949232.053273:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289683200, offset 0 00000400:00000200:3.0:1784949232.053288:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949232.053295:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949232.053303:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949232.053305:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784949232.053307:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949232.053314:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949232.053320:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 806 00000400:00000200:3.0:1784949232.053327:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1784949232.053329:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949232.053334:0:342755:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949232.053337:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533741509712 : -115539968041904 : ffff96eac13f5c50) 00000400:00000200:3.0:1784949232.053338:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949232.053342:0:342755:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949232.053352:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1784949232.053354:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000093c39287 x1871651289683072/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:138/0 lens 224/0 e 0 to 0 dl 1784949243 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949232.053364:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949232.053381:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3786:3786:242:4294967295] 192.168.202.137@tcp LPNI seq info [2422:2422:8:4294967295] 00000100:00000001:1.0:1784949232.053384:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949232.053393:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784949232.053403:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949232.053411:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000400:00000200:3.0:1784949232.053413:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:1.0:1784949232.053425:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000093c39287 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289683072:12345-0@lo:400: 00000800:00000200:3.0:1784949232.053435:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000200:1.0:1784949232.053437:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289683072 00000020:00000001:1.0:1784949232.053446:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000010:3.0:1784949232.053448:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000020:00000001:1.0:1784949232.053452:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949232.053457:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949232.053465:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000200:3.0:1784949232.053466:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:1.0:1784949232.053470:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949232.053477:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949232.053481:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949232.053484:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:1.0:1784949232.053487:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949232.053494:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000200:3.0:1784949232.053496:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000020:00000001:1.0:1784949232.053499:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949232.053507:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949232.053512:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949232.053515:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949232.053521:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000077ad1613. 02000000:00000001:1.0:1784949232.053525:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949232.053527:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949232.053528:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949232.053538:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949232.053540:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2225 usecs (3 jiffies) 00010000:00000040:1.0:1784949232.053543:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289683072 00000100:00000001:3.0:1784949232.053548:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306171396 00010000:00000001:1.0:1784949232.053550:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949232.053559:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306171435 00010000:00000200:1.0:1784949232.053563:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000093c39287 x1871651289683072/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:138/0 lens 224/224 e 0 to 0 dl 1784949243 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949232.053567:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306171396 00000100:00000001:3.0:1784949232.053577:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306171435 00000100:00000001:3.0:1784949232.053587:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306171396 00010000:00000001:1.0:1784949232.053590:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949232.053596:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306171435 00010000:00000001:1.0:1784949232.053597:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949232.053605:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=104 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949232.053608:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306171396 00000100:00000001:1.0:1784949232.053615:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949232.053619:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306171435 00000100:00000040:1.0:1784949232.053621:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949232.053627:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949232.053633:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949232.053634:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949232.053639:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949232.053645:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949232.053647:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949232.053648:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000010:0.0:1784949232.053658:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000100:00000040:1.0:1784949232.053659:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949232.053660:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949232.053668:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000001:3.0:1784949232.053675:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949232.053676:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1784949232.053677:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289683072, offset 224 00000400:00000200:1.0:1784949232.053700:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:2.0:1784949232.053703:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949232.053707:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949232.053720:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000400:00000200:1.0:1784949232.053721:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:2.0:1784949232.053725:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949232.053732:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000800:00000001:2.0:1784949232.053736:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949232.053745:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d480 00000100:00000001:0.0:1784949232.053746:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:2.0:1784949232.053751:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949232.053761:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949232.053762:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9329 [1] + 224 00000400:00000200:2.0:1784949232.053779:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949232.053785:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949232.053802:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949232.053806:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949232.053812:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1784949232.053814:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d500 00000100:00000040:0.0:1784949232.053819:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949232.053823:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949232.053847:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 176208 00000800:00000001:2.0:1784949232.053873:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949232.053874:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.053885:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784949232.053891:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949232.053891:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949232.053967:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949232.053977:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949232.053985:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949232.053990:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000400:00000010:1.0:1784949232.053995:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000100:00000001:1.0:1784949232.053999:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949232.054002:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949232.054008:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949232.054009:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949232.054017:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:1.0:1784949232.054020:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949232.054057:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1784949232.054058:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000010:2.0:1784949232.054061:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fcdc5039. 00000020:00000001:1.0:1784949232.054066:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949232.054066:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784949232.054073:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000fcdc5039 x1871651289683200 msgsize 224 00000020:00000001:1.0:1784949232.054075:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:3.0:1784949232.054079:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949232.054085:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1784949232.054086:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000093c39287 x1871651289683072/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:138/0 lens 224/224 e 0 to 0 dl 1784949243 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949232.054093:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008c614621. 00000100:00100000:2.0:1784949232.054104:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:3.0:1784949232.054108:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000400:00000010:3.0:1784949232.054118:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000100:00100000:1.0:1784949232.054127:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000093c39287 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289683072:12345-0@lo:400: Request processed in 736us (1354us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949232.054130:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949232.054141:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949232.054144:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00100000:1.0:1784949232.054147:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 806 00000100:00000040:1.0:1784949232.054160:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000800:00000001:2.0:1784949232.054164:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949232.054168:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949232.054178:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:2.0:1784949232.054179:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949232.054185:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949232.054188:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000010:1.0:1784949232.054189:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000071ebe728. 00000100:00100000:0.0:1784949232.054196:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289683200 02000000:00000001:0.0:1784949232.054205:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000010:1.0:1784949232.054205:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949232.054210:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949232.054212:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949232.054221:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949232.054222:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000100:00000001:3.0:1784949232.054222:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949232.054233:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949232.054235:0:337119:0:(client.c:1580:after_reply()) Process entered 00000020:00000040:1.0:1784949232.054240:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:3.0:1784949232.054243:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00100000:0.0:1784949232.054244:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289683200 00000100:00000001:3.0:1784949232.054250:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:0.0:1784949232.054253:0:343544:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784949232.054255:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949232.054256:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949232.054258:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000010:1.0:1784949232.054263:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000077ad1613. 00000020:00000001:0.0:1784949232.054267:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1784949232.054268:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949232.054275:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:1.0:1784949232.054276:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:3.0:1784949232.054278:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=104 v=5 (5 5 5 5) 00000020:00000001:0.0:1784949232.054286:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949232.054288:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784949232.054289:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=104 v=5 (1 1 1 1) 00000020:00000001:0.0:1784949232.054292:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784949232.054308:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949232.054309:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949232.054310:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949232.054317:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949232.054321:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000400:00000010:1.0:1784949232.054323:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b4e968f8. 00010000:00000001:3.0:1784949232.054324:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784949232.054328:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784949232.054331:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949232.054339:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000400:00000010:1.0:1784949232.054339:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008c74c22d. 00000100:00000001:3.0:1784949232.054341:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949232.054355:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949232.054357:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000020:00000010:0.0:1784949232.054369:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bdafdeaa. 00000100:00000001:3.0:1784949232.054374:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949232.054378:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949232.054380:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949232.054386:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949232.054392:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949232.054394:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949232.054403:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949232.054407:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00100000:3.0:1784949232.054413:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289683072:0@lo:400: 00000100:00000001:0.0:1784949232.054414:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949232.054427:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949232.054428:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949232.054433:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949232.054435:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949232.054440:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949232.054445:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b16a92e x1871651289683072/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949232.054457:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949232.054459:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1784949232.054463:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784949232.054464:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000089ec28da. 02000000:00000001:3.0:1784949232.054475:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949232.054480:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949232.054483:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949232.054484:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949232.054491:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000001:3.0:1784949232.054493:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949232.054498:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000635133da. 00000100:00100000:0.0:1784949232.054505:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 782 02000000:00000001:3.0:1784949232.054507:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:0.0:1784949232.054511:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 02000000:00000001:3.0:1784949232.054513:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949232.054520:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b16a92e. 00000100:00000001:0.0:1784949232.054521:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562010496 : -115544147541120 : ffff96e9c8214380) 00000100:00000001:3.0:1784949232.054533:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:0.0:1784949232.054536:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000fcdc5039 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:138/0 lens 224/0 e 0 to 0 dl 1784949243 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949232.054539:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949232.054549:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949232.054562:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949232.054564:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1784949232.054567:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e1ce918d. 00000100:00000001:3.0:1784949232.054569:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.054573:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949232.054576:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:0.0:1784949232.054580:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e1ce918d. 00000100:00000001:3.0:1784949232.054582:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949232.054590:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000fcdc5039 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289683200:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949232.054596:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289683200 00000020:00000001:0.0:1784949232.054599:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949232.054603:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949232.054606:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949232.054609:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949232.054611:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949232.054615:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949232.054618:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949232.054621:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949232.054623:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949232.054626:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949232.054628:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949232.054632:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949232.054634:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949232.054638:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000f3ecd62c. 02000000:00000001:0.0:1784949232.054641:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.054643:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949232.054646:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949232.054649:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289683200 00010000:00000001:0.0:1784949232.054653:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949232.054658:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000fcdc5039 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:138/0 lens 224/224 e 0 to 0 dl 1784949243 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949232.054674:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949232.054678:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949232.054682:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=99 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949232.054689:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949232.054694:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949232.054698:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949232.054703:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949232.054705:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949232.054709:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949232.054714:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949232.054719:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000dd8634bc. 00000100:00000200:0.0:1784949232.054726:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289683200, offset 224 00000400:00000200:0.0:1784949232.054735:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949232.054747:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949232.054753:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3787:3787:242:4294967295] 192.168.202.137@tcp LPNI seq info [2423:2423:8:4294967295] 00000400:00000200:0.0:1784949232.054765:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949232.054771:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949232.054775:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003a8a98bc. 00000800:00000200:0.0:1784949232.054781:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949232.054787:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949232.054792:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003a8a98bc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949232.054812:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949232.054816:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949232.054819:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949232.054821:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949232.054824:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949232.054829:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcdc5039 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:138/0 lens 224/224 e 0 to 0 dl 1784949243 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949232.054841:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000fcdc5039 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289683200:12345-192.168.202.137@tcp:400: Request processed in 278us (767us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949232.054854:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 782 00000100:00000040:0.0:1784949232.054858:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949232.054861:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949232.054865:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949232.054869:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bdafdeaa. 00000020:00000010:0.0:1784949232.054874:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:0.0:1784949232.054877:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000020:00000040:0.0:1784949232.054887:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949232.054891:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949232.054991:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949232.055003:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949232.055003:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949232.055014:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949232.055015:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000800:00000001:0.0:1784949232.055020:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949232.055023:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949232.055053:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949232.055071:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949232.055078:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d500 00000400:00000200:3.0:1784949232.055083:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000dd8634bc 00000400:00000010:3.0:1784949232.055089:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000dd8634bc. 00000100:00000001:3.0:1784949232.055094:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949232.055097:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784949232.055102:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9331 [1] + 224 00000800:00000001:0.0:1784949232.055112:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949232.055126:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949232.055130:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949232.055134:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949232.055140:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949232.055145:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949232.055155:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949232.055176:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949232.055183:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949232.055186:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949232.055229:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949232.055233:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949232.055239:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949232.055241:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949232.055243:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000400:00000200:3.0:1784949232.055251:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000400:00000010:3.0:1784949232.055253:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000100:00000001:3.0:1784949232.055256:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949232.055262:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949232.055273:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949232.055282:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949232.055284:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949232.055286:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949232.055289:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949232.055290:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949232.055292:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949232.055294:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949232.055300:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=99 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949232.055304:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=99 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949232.055307:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949232.055308:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949232.055310:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949232.055314:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949232.055324:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949232.055328:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949232.055330:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949232.055332:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949232.055334:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949232.055337:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949232.055345:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949232.055347:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949232.055348:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949232.055351:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949232.055360:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a3e309c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289683200:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949232.055364:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949232.055366:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949232.055369:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a3e309c2 x1871651289683200/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949232.055377:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949232.055379:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949232.055381:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000072ce5ef0. 02000000:00000001:3.0:1784949232.055385:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949232.055386:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949232.055388:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949232.055391:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949232.055393:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000057c7ab4. 02000000:00000001:3.0:1784949232.055396:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949232.055397:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949232.055399:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a3e309c2. 00000100:00000001:3.0:1784949232.055403:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949232.055405:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949232.055407:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949232.055412:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949232.055414:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949232.055416:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949232.055418:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949232.095998:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306171435, at 4306171436 00000100:00000010:0.0:1784949232.096015:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00000100:00000040:1.0:1784949232.096018:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306171435, at 4306171436 00000100:00000040:2.0:1784949232.096025:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306171435, at 4306171436 00000100:00000040:3.0:1784949232.096025:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306171435, at 4306171436 00000100:00000010:1.0:1784949232.096049:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000010de4e21. 00000100:00000010:3.0:1784949232.096058:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a2179890. 00000100:00000010:2.0:1784949232.096085:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00010000:00000001:2.0:1784949232.884082:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949232.884094:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949232.884106:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949232.884109:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949232.884239:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949232.884246:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949232.884251:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949233.076011:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949233.076064:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949233.076067:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949233.076073:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949233.076077:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949233.076082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949233.076085:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949233.908034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949233.908042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949233.908050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949233.908051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949233.908104:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949233.908109:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949233.908113:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949234.100127:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949234.100145:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949234.100148:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949234.100156:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949234.100161:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949234.100165:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949234.100169:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949234.932058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949234.932067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949234.932075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949234.932077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949234.932177:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949234.932186:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949234.932192:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949235.124058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949235.124075:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949235.124077:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949235.124083:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949235.124087:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949235.124092:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949235.124095:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949235.956074:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949235.956083:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949235.956091:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949235.956093:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949235.956221:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949235.956227:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949235.956232:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949236.148016:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949236.148026:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784949236.148044:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949211/real 1784949211] req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949236 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949236.148070:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949236.148074:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949236.148080:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000075be34c1. 00000400:00000200:1.0:1784949236.148088:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f3654321 00000400:00000010:1.0:1784949236.148091:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f3654321. 00000100:00000001:1.0:1784949236.148097:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949236.148101:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949236 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949236.148113:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949236 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949236.148123:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949236.148125:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148129:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949236.148131:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949236.148135:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949236 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949236.148143:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148153:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949236.148155:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.148159:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949236.148162:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949236.148165:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949236.148169:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949236 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949236.148178:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949236.148182:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148184:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949236.148185:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148188:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784949236.148191:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784949236.148195:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784949236.148222:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:1.0:1784949236.148225:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784949236.148231:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949236 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949236.148241:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289682048:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784949236.148248:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148251:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949236.148254:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289682048/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949236 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949236.148262:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949236.148266:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949236.148271:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000000b08bf97. 02000000:00000001:1.0:1784949236.148277:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949236.148279:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949236.148281:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:1.0:1784949236.148284:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949236.148287:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000579609a4. 02000000:00000001:1.0:1784949236.148292:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949236.148294:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949236.148296:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000100:00000001:1.0:1784949236.148315:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949236.148317:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148320:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148324:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949236.148327:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.148330:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949236.148332:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949236.148341:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949236.148351:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949236.148360:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000c6c2c60. 00000020:00000040:0.0:1784949236.148367:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949236.148373:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949236.148377:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949236.148383:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949236.148391:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000009ff40a64. 00000100:00000001:0.0:1784949236.148400:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949236.148409:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949236.148422:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949236.148426:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949236.148432:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c6c2c60] to pc [ptlrpcd_00_03+3] req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949236.148470:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949236.148473:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949236.148478:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949236.148481:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949236.148483:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949236.148487:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949236.148494:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11483 00000400:00000200:0.0:1784949236.148506:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784949236.148512:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11482 00000100:00000001:1.0:1784949236.148516:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949236.148524:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:1.0:1784949236.148527:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949236.148532:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949236.148535:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949236.148537:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:1.0:1784949236.148541:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949236.148548:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:1.0:1784949236.148549:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949236.148556:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949236.148561:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949236.148566:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949236.148567:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949236.148571:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949236.148572:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000040:1.0:1784949236.148581:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949236.148582:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949236.148582:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949236.148588:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784949236.148596:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949236.148604:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:1.0:1784949236.148608:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1784949236.148610:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:1.0:1784949236.148613:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949236.148616:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784949236.148621:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949236.148623:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949236.148629:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949236.148630:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949236.148637:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949236.148638:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949236.148649:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:2.0:1784949236.148652:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00100000:1.0:1784949236.148653:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000c6c2c60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289683328:0@lo:400: 00000020:00000040:0.0:1784949236.148656:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:1.0:1784949236.148669:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1784949236.148670:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00080000:0.0:1784949236.148671:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000001:1.0:1784949236.148680:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1784949236.148685:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 02000000:00000001:1.0:1784949236.148687:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949236.148689:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 00000100:00000001:0.0:1784949236.148694:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949236.148698:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1784949236.148701:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1784949236.148704:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 02000000:00000010:1.0:1784949236.148707:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004f865a97. 02000000:00000001:0.0:1784949236.148712:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949236.148714:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949236.148718:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001f2bdc7c. 02000000:00000001:0.0:1784949236.148723:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949236.148729:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000839265ff. 00000100:00000001:0.0:1784949236.148730:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1784949236.148731:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784949236.148740:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949236.148749:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eac1e2f5. 00000400:00000010:1.0:1784949236.148750:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000020:00000040:0.0:1784949236.148755:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:2.0:1784949236.148758:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000200:1.0:1784949236.148765:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289683328, portal 25 02000000:00000001:0.0:1784949236.148767:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1784949236.148770:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784949236.148773:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.148777:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949236.148781:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1784949236.148782:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469566336 : -115544239985280 : ffff96e9c29ead80) 00000400:00000200:2.0:1784949236.148784:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:0.0:1784949236.148787:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949236.148794:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949236.148799:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1784949236.148800:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949236.148808:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000be80dc96. 00000400:00000200:2.0:1784949236.148811:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000001:0.0:1784949236.148820:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.148827:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784949236.148829:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3788:3788:242:4294967295] 1.2.3.4@tcp LPNI seq info [116:116:8:4294967295] 00000100:00100000:0.0:1784949236.148838:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1784949236.148840:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949236.148851:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000200:2.0:1784949236.148854:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000200:1.0:1784949236.148863:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289683328, offset 0 00000100:00000001:0.0:1784949236.148873:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949236.148880:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949236.148884:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949236.148886:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000040:0.0:1784949236.148894:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eac1e2f5] to pc [ptlrpcd_rcv+-1] req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949236.148954:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:2.0:1784949236.148962:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000800:00000010:2.0:1784949236.148977:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000400:00000200:1.0:1784949236.148985:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d580 00000100:00000001:0.0:1784949236.148992:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949236.149023:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8931 [2] + 0 00000800:00000200:2.0:1784949236.149025:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:1.0:1784949236.149034:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000acd175a6. 00000100:00000001:3.0:1784949236.149035:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949236.149038:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949236.149044:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949236.149048:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000000575a8ab 00000800:00000200:2.1:1784949236.149048:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:3.0:1784949236.149052:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949236.149055:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949236.149065:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000925b9f50. 00000100:00000040:3.0:1784949236.149067:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949236.149068:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:0.0:1784949236.149073:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000400:00000200:2.0:1784949236.149086:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:0.0:1784949236.149087:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1784949236.149089:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784949236.149096:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1784949236.149101:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 02000000:00000001:0.0:1784949236.149104:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949236.149108:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000575a8ab 00000400:00000010:1.0:1784949236.149119:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000000575a8ab. 00000100:00000001:3.0:1784949236.149121:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1784949236.149122:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000034aacccc. 00000100:00000001:1.0:1784949236.149128:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949236.149129:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949236.149131:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949236.149136:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:3.0:1784949236.149140:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784949236.149147:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000d416d8c1 x1871651289683328 msgsize 224 02000000:00000001:3.0:1784949236.149147:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949236.149149:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1784949236.149160:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949236.149177:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1784949236.149178:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eac1e2f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289683456:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949236.149189:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1784949236.149190:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949236.149196:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.149205:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949236.149210:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000925b9f50] to pc [ptlrpcd_00_00+0] req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949236.149211:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949236.149217:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949236.149225:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000100:00000001:3.0:1784949236.149228:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949236.149234:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000100:00000001:3.0:1784949236.149239:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.149246:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949236.149248:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949236.149257:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949236.149259:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949236.149265:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949236.149269:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.149275:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949236.149282:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11513) 00000100:00000001:3.0:1784949236.149283:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949236.149292:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.149293:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949236.149302:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949236.149304:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949236.149319:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.149322:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949236.149326:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949236.149331:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.149331:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949236.149333:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289683328 00000100:00000001:3.0:1784949236.149339:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949236.149343:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949236.149344:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949236.149345:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949236.149346:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306175531 00000100:00000001:0.0:1784949236.149353:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949236.149354:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949236.149358:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000040:3.0:1784949236.149360:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949236.149362:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949236.149375:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.149377:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00100000:0.0:1784949236.149389:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289683328 00000100:00000001:3.0:1784949236.149390:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949236.149393:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306175531 00000100:00000001:3.0:1784949236.149397:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949236.149398:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000010:1.0:1784949236.149404:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000040:0.0:1784949236.149405:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 02000000:00000001:3.0:1784949236.149407:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949236.149414:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1784949236.149416:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1784949236.149417:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949236.149423:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306175531 00000020:00000040:0.0:1784949236.149425:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 02000000:00000001:3.0:1784949236.149426:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949236.149435:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000020:00000001:0.0:1784949236.149436:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949236.149444:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000001:0.0:1784949236.149445:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949236.149456:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949236.149462:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000100:00000001:0.0:1784949236.149466:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949236.149467:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306175531 00000020:00000010:0.0:1784949236.149470:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000100:00000010:1.0:1784949236.149478:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000010:0.0:1784949236.149484:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:1.0:1784949236.149487:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000010:0.0:1784949236.149497:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000005e55c8b7. 00000100:00000001:1.0:1784949236.149498:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949236.149508:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949236.149511:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:1.0:1784949236.149513:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949236.149520:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949236.149530:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00100000:3.0:1784949236.149531:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000925b9f50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289683584:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949236.149540:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949236.149542:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949236.149547:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784949236.149550:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949236.149552:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949236.149556:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949236.149561:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949236.149566:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949236.149568:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 02000000:00000001:3.0:1784949236.149574:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949236.149580:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000400:00000010:3.0:1784949236.149588:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 00000100:00000001:0.0:1784949236.149589:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949236.149598:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289683584, portal 10 00000100:00000001:3.0:1784949236.149604:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949236.149607:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469565440 : -115544239986176 : ffff96e9c29eaa00) 00000100:00000001:0.0:1784949236.149611:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949236.149616:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:3.0:1784949236.149618:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949236.149629:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 807 00000100:00000040:0.0:1784949236.149642:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949236.149645:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949236.149649:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466828880 : -115544242722736 : ffff96e9c274e850) 00000100:00000040:3.0:1784949236.149658:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949236.149663:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000100:00000040:0.0:1784949236.149664:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d416d8c1 x1871651289683328/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:142/0 lens 224/0 e 0 to 0 dl 1784949247 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949236.149671:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289683584, offset 0 00000400:00000200:3.0:1784949236.149690:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949236.149691:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949236.149698:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f6eaba31. 00000400:00000200:3.0:1784949236.149704:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949236.149706:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949236.149715:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f6eaba31. 00000400:00000200:3.0:1784949236.149716:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949236.149726:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00100000:0.0:1784949236.149732:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d416d8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289683328:12345-0@lo:400: 00000400:00000200:3.0:1784949236.149736:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949236.149748:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000200:0.0:1784949236.149749:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289683328 00000020:00000001:0.0:1784949236.149756:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:3.0:1784949236.149757:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3789:3789:241:4294967295] 192.168.202.137@tcp LPNI seq info [2424:2424:8:4294967295] 00000020:00000001:0.0:1784949236.149764:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949236.149769:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949236.149778:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:3.0:1784949236.149782:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:0.0:1784949236.149784:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949236.149793:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949236.149796:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949236.149804:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949236.149805:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949236.149810:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949236.149815:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949236.149820:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000010:3.0:1784949236.149821:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000100:00000001:0.0:1784949236.149823:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949236.149829:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949236.149836:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002f80d253. 00000800:00000200:3.0:1784949236.149838:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:0.0:1784949236.149841:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949236.149852:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949236.149855:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949236.149859:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289683328 00010000:00000001:0.0:1784949236.149868:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000200:3.0:1784949236.149868:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00010000:00000200:0.0:1784949236.149878:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d416d8c1 x1871651289683328/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:142/0 lens 224/224 e 0 to 0 dl 1784949247 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949236.149883:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00010000:00000001:0.0:1784949236.149902:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949236.149935:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949236.149942:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=108 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949236.149947:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949236.149950:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949236.149953:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949236.149957:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949236.149958:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949236.149964:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949236.149969:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2226 usecs (3 jiffies) 00000100:00000001:0.0:1784949236.149970:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949236.149979:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949236.149980:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306175493 00000400:00000010:0.0:1784949236.149985:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00000001:3.0:1784949236.149987:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306175531 00000100:00000200:0.0:1784949236.150020:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289683328, offset 224 00000100:00000001:3.0:1784949236.150023:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306175493 00000100:00000001:3.0:1784949236.150031:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306175531 00000400:00000200:0.0:1784949236.150032:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949236.150041:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306175493 00000100:00000001:3.0:1784949236.150046:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306175531 00000100:00000001:3.0:1784949236.150050:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306175493 00000100:00000001:3.0:1784949236.150056:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306175531 00000400:00000200:0.0:1784949236.150057:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949236.150063:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949236.150068:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.150072:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949236.150078:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d580 00000100:00000001:3.0:1784949236.150079:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949236.150095:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9349 [1] + 224 00000100:00000001:3.0:1784949236.150095:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949236.150101:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784949236.150107:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949236.150110:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949236.150111:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949236.150116:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949236.150124:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949236.150125:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949236.150137:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949236.150145:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949236.150159:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949236.150177:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949236.150186:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000100:00000001:0.0:1784949236.150187:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949236.150197:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949236.150202:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:0.0:1784949236.150210:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000800:00000001:1.0:1784949236.150211:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949236.150212:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949236.150217:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949236.150225:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949236.150227:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000100:00000001:0.0:1784949236.150232:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949236.150234:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000800:00000001:1.0:1784949236.150235:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949236.150245:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949236.150248:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784949236.150249:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000800:00000001:1.0:1784949236.150255:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949236.150256:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784949236.150257:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:2.0:1784949236.150262:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949236.150265:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784949236.150266:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949236.150278:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:3.0:1784949236.150279:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1784949236.150284:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000001:3.0:1784949236.150287:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1784949236.150293:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d416d8c1 x1871651289683328/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:142/0 lens 224/224 e 0 to 0 dl 1784949247 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949236.150296:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000839265ff. 00000400:00000200:3.0:1784949236.150310:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000400:00000010:3.0:1784949236.150320:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000400:00000200:1.0:1784949236.150321:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d680 00000100:00000001:2.0:1784949236.150325:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949236.150333:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1784949236.150341:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949236.150349:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 176432 00000100:00000200:3.0:1784949236.150349:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949236.150353:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d416d8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289683328:12345-0@lo:400: Request processed in 653us (1197us total) trans 0 rc 0/0 00000800:00000001:1.0:1784949236.150376:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949236.150388:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949236.150398:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 807 00000400:00000200:1.0:1784949236.150401:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949236.150407:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949236.150414:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949236.150417:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784949236.150424:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:3.0:1784949236.150425:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949236.150426:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949236.150435:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:1.0:1784949236.150443:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000020:00000010:0.0:1784949236.150445:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000005e55c8b7. 00000100:00000200:1.0:1784949236.150450:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784949236.150450:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784949236.150456:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949236.150458:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784949236.150461:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000047bf8f5f. 00000100:00000001:3.0:1784949236.150468:0:337121:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:0.0:1784949236.150470:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000100:00000040:1.0:1784949236.150471:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000047bf8f5f x1871651289683584 msgsize 224 02000000:00000001:3.0:1784949236.150475:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949236.150482:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:0.0:1784949236.150485:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949236.150488:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949236.150496:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00100000:1.0:1784949236.150497:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:3.0:1784949236.150499:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949236.150503:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000002f80d253. 02000000:00000001:0.0:1784949236.150511:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:3.0:1784949236.150512:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=108 v=5 (5 5 5 5) 00000100:00000001:0.0:1784949236.150515:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949236.150528:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:3.0:1784949236.150528:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=108 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949236.150529:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949236.150541:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:0.0:1784949236.150542:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000faa31c82. 00000800:00000001:1.0:1784949236.150544:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949236.150545:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949236.150554:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000800:00000001:1.0:1784949236.150555:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949236.150556:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f27a392e. 00010000:00000001:3.0:1784949236.150559:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.150569:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949236.150577:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949236.150586:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949236.150588:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784949236.150594:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289683584 00000100:00000001:3.0:1784949236.150595:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949236.150602:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949236.150603:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949236.150608:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784949236.150613:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949236.150614:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949236.150622:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949236.150631:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289683584 00000020:00000001:2.0:1784949236.150638:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000100:00100000:3.0:1784949236.150642:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000c6c2c60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289683328:0@lo:400: 00000020:00000040:2.0:1784949236.150644:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949236.150651:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949236.150655:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1784949236.150657:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:3.0:1784949236.150662:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949236.150664:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:3.0:1784949236.150670:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000c6c2c60 x1871651289683328/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949236.150672:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949236.150683:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949236.150691:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:3.0:1784949236.150693:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949236.150699:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1784949236.150700:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1784949236.150704:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004f865a97. 00000020:00000010:2.0:1784949236.150707:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 02000000:00000001:3.0:1784949236.150712:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949236.150716:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000010:2.0:1784949236.150719:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000098dd29a0. 00000020:00000040:3.0:1784949236.150720:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949236.150729:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000010:2.0:1784949236.150731:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000738a21fc. 02000000:00000010:3.0:1784949236.150732:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000009ff40a64. 02000000:00000001:3.0:1784949236.150745:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000010:2.0:1784949236.150746:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 02000000:00000001:3.0:1784949236.150750:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949236.150755:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000c6c2c60. 00000100:00000040:2.0:1784949236.150759:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949236.150763:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949236.150766:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949236.150768:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949236.150771:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949236.150774:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949236.150777:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949236.150781:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949236.150786:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949236.150787:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949236.150793:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949236.150797:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949236.150798:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.150807:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949236.150821:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949236.150839:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949236.150842:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949236.150850:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 783 00000100:00000040:2.0:1784949236.150854:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949236.150858:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637805568 : -115544071746048 : ffff96e9cca5ce00) 00000100:00000040:2.0:1784949236.150865:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000047bf8f5f x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:142/0 lens 224/0 e 0 to 0 dl 1784949247 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949236.150878:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949236.150882:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000066052e0b. 00000100:00000001:2.0:1784949236.150886:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949236.150889:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000066052e0b. 00000100:00100000:2.0:1784949236.150894:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000047bf8f5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289683584:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784949236.150900:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289683584 00000020:00000001:2.0:1784949236.150902:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949236.150939:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949236.150942:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949236.150945:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949236.150947:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949236.150950:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949236.150954:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949236.150956:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949236.150958:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949236.150961:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949236.150963:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949236.150967:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949236.150969:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949236.150974:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000094890ca4. 02000000:00000001:2.0:1784949236.150976:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949236.150979:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949236.150981:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949236.150984:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289683584 00010000:00000001:2.0:1784949236.150986:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949236.150990:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000047bf8f5f x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:142/0 lens 224/224 e 0 to 0 dl 1784949247 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949236.151028:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949236.151031:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949236.151034:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=103 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949236.151039:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949236.151042:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949236.151044:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949236.151048:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949236.151049:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949236.151052:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949236.151055:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949236.151059:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000200:2.0:1784949236.151063:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289683584, offset 224 00000400:00000200:2.0:1784949236.151070:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949236.151080:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949236.151086:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3790:3790:241:4294967295] 192.168.202.137@tcp LPNI seq info [2425:2425:8:4294967295] 00000400:00000200:2.0:1784949236.151098:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949236.151104:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949236.151108:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000800:00000200:2.0:1784949236.151113:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949236.151120:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949236.151124:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949236.151140:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949236.151143:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949236.151146:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949236.151148:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949236.151150:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949236.151155:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000047bf8f5f x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:142/0 lens 224/224 e 0 to 0 dl 1784949247 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949236.151165:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000047bf8f5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289683584:12345-192.168.202.137@tcp:400: Request processed in 287us (695us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949236.151174:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 783 00000100:00000040:2.0:1784949236.151178:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949236.151181:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949236.151182:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949236.151186:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000738a21fc. 00000020:00000010:2.0:1784949236.151190:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949236.151194:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000098dd29a0. 00000020:00000040:2.0:1784949236.151198:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949236.151201:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949236.151354:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949236.151361:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000400:00000200:0.0:1784949236.151370:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949236.151371:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949236.151380:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784949236.151386:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949236.151388:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000800:00000001:2.0:1784949236.151395:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:0.0:1784949236.151396:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:0.0:1784949236.151406:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784949236.151407:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949236.151411:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949236.151422:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949236.151436:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d680 00000400:00000200:2.0:1784949236.151444:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9351 [1] + 224 00000800:00000001:2.0:1784949236.151451:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949236.151461:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949236.151464:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949236.151468:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949236.151473:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949236.151477:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949236.151489:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949236.151510:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949236.151516:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949236.151519:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949236.151616:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949236.151622:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949236.151629:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949236.151631:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949236.151634:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000400:00000200:3.0:1784949236.151643:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000400:00000010:3.0:1784949236.151645:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000100:00000001:3.0:1784949236.151649:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949236.151657:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949236.151670:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949236.151680:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949236.151682:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949236.151685:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949236.151687:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949236.151689:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949236.151692:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949236.151695:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949236.151701:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=103 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949236.151705:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=103 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949236.151708:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949236.151710:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949236.151712:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949236.151716:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949236.151726:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.151729:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949236.151731:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949236.151733:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949236.151734:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949236.151738:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949236.151746:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949236.151750:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949236.151752:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949236.151758:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949236.151774:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000925b9f50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289683584:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949236.151783:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949236.151786:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949236.151790:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000925b9f50 x1871651289683584/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949252 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949236.151804:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949236.151807:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949236.151811:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 02000000:00000001:3.0:1784949236.151819:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949236.151821:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949236.151824:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949236.151829:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949236.151832:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000034aacccc. 02000000:00000001:3.0:1784949236.151836:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949236.151838:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949236.151841:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000925b9f50. 00000100:00000001:3.0:1784949236.151846:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949236.151848:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949236.151852:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949236.151860:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949236.151863:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949236.151866:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949236.151869:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949236.192032:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306175531, at 4306175532 00000100:00000010:1.0:1784949236.192047:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:2.0:1784949236.192052:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306175531, at 4306175532 00000100:00000010:2.0:1784949236.192067:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000040:0.0:1784949236.192090:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306175531, at 4306175532 00000100:00000040:3.0:1784949236.192095:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306175531, at 4306175532 00000100:00000010:0.0:1784949236.192105:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000010:3.0:1784949236.192113:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949236.980068:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949236.980077:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949236.980086:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949236.980088:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949236.980167:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949236.980173:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949236.980177:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949237.172132:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949237.172143:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784949237.172161:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949236/real 0] req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949237 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.172182:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949237.172186:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949237.172189:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949237.172194:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949237 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.172205:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949237.172217:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949237.172219:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949237.172224:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949237.172227:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949237.172230:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949237.172234:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949237 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.172243:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949237.172247:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949237.172249:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949237.172251:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949237.172254:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784949237.172256:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784949237.172260:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784949237.172288:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:1.0:1784949237.172292:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784949237.172296:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949237 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949237.172305:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eac1e2f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289683456:1.2.3.4@tcp:8: 00000100:00000001:1.0:1784949237.172312:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949237.172316:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949237.172319:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eac1e2f5 x1871651289683456/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949237 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.172329:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1784949237.172333:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949237.172336:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:1.0:1784949237.172339:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949237.172344:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000be80dc96. 00000100:00000040:0.0:1784949237.172350:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1784949237.172351:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949237.172366:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949237.172373:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eac1e2f5. 00000100:00000001:0.0:1784949237.172374:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949237.172384:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000010:0.0:1784949237.172386:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003bf8e8ad. 00000100:00000001:1.0:1784949237.172387:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1784949237.172398:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949237.172398:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949237.172403:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949237.172409:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949237.172410:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949237.172417:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949237.172418:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949237.172423:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949237.172427:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949237.172440:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949237.172451:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949237.172464:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949237.172485:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949237.172490:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949237.172498:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf8e8ad] to pc [ptlrpcd_00_01+1] req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949237.172547:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949237.172552:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949237.172558:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949237.172562:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949237.172565:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949237.172571:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949237.172578:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11483 00000100:00000001:3.0:1784949237.172590:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949237.172591:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:0.0:1784949237.172601:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11508 00000100:00100000:3.0:1784949237.172602:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1784949237.172604:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949237.172613:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949237.172614:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:2.0:1784949237.172617:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949237.172625:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1784949237.172628:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949237.172635:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949237.172636:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784949237.172646:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949237.172647:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949237.172652:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949237.172658:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949237.172659:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949237.172668:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000040:3.0:1784949237.172672:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949237.172676:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949237.172683:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784949237.172691:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949237.172698:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:3.0:1784949237.172699:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949237.172706:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949237.172707:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784949237.172715:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1784949237.172716:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784949237.172723:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949237.172727:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000001:3.0:1784949237.172730:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949237.172732:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1784949237.172738:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289683712:0@lo:400: 00000100:00000040:0.0:1784949237.172741:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949237.172751:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784949237.172752:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1784949237.172756:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000001:3.0:1784949237.172762:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1784949237.172764:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 02000000:00000001:3.0:1784949237.172769:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949237.172773:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 02000000:00000001:3.0:1784949237.172777:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949237.172782:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949237.172787:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1784949237.172787:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 02000000:00000001:3.0:1784949237.172797:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949237.172798:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949237.172808:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949237.172810:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784949237.172814:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949237.172814:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061b5c7fd. 00000100:00000010:0.0:1784949237.172824:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bab47beb. 00000020:00000040:0.0:1784949237.172830:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000010:3.0:1784949237.172834:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 02000000:00000001:0.0:1784949237.172836:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949237.172841:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949237.172848:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949237.172850:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949237.172854:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949237.172854:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289683712, portal 25 00000100:00000001:3.0:1784949237.172865:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000010:0.0:1784949237.172866:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000efbd7ee7. 00000100:00000001:3.0:1784949237.172872:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469562752 : -115544239988864 : ffff96e9c29e9f80) 00000100:00000001:0.0:1784949237.172876:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949237.172893:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949237.172894:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949237.172945:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949237.172955:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949237.172963:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000100:00000001:0.0:1784949237.172968:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1784949237.172970:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289683712, offset 0 00000100:00000001:0.0:1784949237.172973:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949237.173014:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bab47beb] to pc [ptlrpcd_rcv+-1] req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949237.173014:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949237.173053:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949237.173055:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:0.0:1784949237.173061:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949237.173069:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949237.173075:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000aedb1960. 00000400:00000200:3.0:1784949237.173076:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d700 00000020:00000040:0.0:1784949237.173085:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949237.173089:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949237.173091:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1784949237.173096:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x894d [2] + 0 02000000:00000001:0.0:1784949237.173098:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949237.173100:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949237.173107:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949237.173108:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:3.0:1784949237.173110:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004a584a71. 02000000:00000010:0.0:1784949237.173120:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005a1c05a3. 00000400:00000200:3.0:1784949237.173121:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d5e12ff0 00000100:00000040:1.0:1784949237.173124:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949237.173134:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949237.173137:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784949237.173150:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1784949237.173155:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949237.173172:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d5e12ff0 00000400:00000010:3.0:1784949237.173180:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d5e12ff0. 00000100:00000001:1.0:1784949237.173183:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949237.173189:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949237.173199:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949237.173205:0:337119:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:1.0:1784949237.173206:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949237.173213:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949237.173215:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949237.173219:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784949237.173220:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949237.173228:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289683840:192.168.202.137@tcp:8: 00000100:00000040:3.0:1784949237.173230:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000ab8619ba x1871651289683712 msgsize 224 00000100:00000040:0.0:1784949237.173236:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aedb1960] to pc [ptlrpcd_00_02+2] req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.173245:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949237.173253:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1784949237.173254:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 02000000:00000001:1.0:1784949237.173258:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949237.173263:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 02000000:00000001:1.0:1784949237.173270:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949237.173281:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000077ad1613. 00000100:00000001:3.0:1784949237.173285:0:337119:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:1.0:1784949237.173287:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949237.173296:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000acd175a6. 00000400:00000200:3.0:1784949237.173299:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000100:00000001:0.0:1784949237.173303:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949237.173305:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000100:00000040:0.0:1784949237.173317:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11514) 00000100:00000001:3.0:1784949237.173317:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:1.0:1784949237.173322:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000200:3.0:1784949237.173330:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949237.173331:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289683840, portal 4 00000100:00000001:1.0:1784949237.173341:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949237.173343:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949237.173347:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469556480 : -115544239995136 : ffff96e9c29e8700) 00000100:00100000:0.0:1784949237.173351:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:0.0:1784949237.173360:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949237.173370:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949237.173383:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949237.173390:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949237.173390:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949237.173402:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289683712 00000100:00000001:0.0:1784949237.173404:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949237.173411:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1784949237.173414:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949237.173417:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949237.173423:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949237.173425:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:0.0:1784949237.173433:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.173436:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949237.173436:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949237.173448:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949237.173461:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784949237.173466:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949237.173470:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000200:1.0:1784949237.173482:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289683840, offset 0 00000100:00100000:2.0:1784949237.173486:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289683712 00000020:00000001:2.0:1784949237.173499:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949237.173502:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949237.173505:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000040:2.0:1784949237.173508:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949237.173513:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949237.173520:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1784949237.173522:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949237.173527:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949237.173529:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000040:2.0:1784949237.173532:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 02000000:00000001:0.0:1784949237.173541:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1784949237.173545:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 02000000:00000001:0.0:1784949237.173549:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1784949237.173553:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784949237.173560:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784949237.173561:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949237.173566:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:0.0:1784949237.173575:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289683968:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784949237.173576:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:2.0:1784949237.173579:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000400:00000200:1.0:1784949237.173589:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949237.173594:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949237.173594:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949237.173599:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:0.0:1784949237.173603:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000010:2.0:1784949237.173607:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 02000000:00000001:0.0:1784949237.173609:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949237.173616:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3791:3791:241:4294967295] 192.168.202.137@tcp LPNI seq info [2426:2426:8:4294967295] 02000000:00000001:0.0:1784949237.173619:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:2.0:1784949237.173626:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000098dd29a0. 02000000:00000010:0.0:1784949237.173629:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 02000000:00000001:0.0:1784949237.173639:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949237.173643:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000738a21fc. 00000400:00000200:1.0:1784949237.173649:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000400:00000010:0.0:1784949237.173653:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001881934. 00000020:00000010:2.0:1784949237.173662:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000800:00000200:1.0:1784949237.173669:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949237.173670:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f594d710. 00000100:00000001:3.0:1784949237.173670:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949237.173683:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949237.173686:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306176555 00000100:00000200:0.0:1784949237.173687:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289683968, portal 10 00000800:00000010:1.0:1784949237.173688:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008d5370ec. 00000100:00000001:2.0:1784949237.173695:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949237.173701:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949237.173703:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000010:3.0:1784949237.173704:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:0.0:1784949237.173707:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469559168 : -115544239992448 : ffff96e9c29e9180) 00000800:00000200:1.0:1784949237.173712:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949237.173714:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.173721:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1784949237.173726:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949237.173742:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.173743:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306176555 00000800:00000200:1.0:1784949237.173749:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000010:3.0:1784949237.173756:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000486261c. 00000800:00000200:1.0:1784949237.173761:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008d5370ec type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1784949237.173767:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949237.173768:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949237.173777:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.173780:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306176555 00000100:00000040:0.0:1784949237.173786:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949237.173789:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949237.173792:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003e9ea21c. 00000400:00000010:0.0:1784949237.173794:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000414ed034. 00000100:00000040:1.0:1784949237.173799:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000200:0.0:1784949237.173805:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289683968, offset 0 00000100:00000001:3.0:1784949237.173807:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949237.173808:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306176555 00000100:00000001:2.0:1784949237.173811:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784949237.173822:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949237.173824:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306176555 00000100:00000001:2.0:1784949237.173825:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949237.173829:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306176555 00000100:00000001:1.0:1784949237.173835:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306176555 00000400:00000200:0.0:1784949237.173837:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000010:3.0:1784949237.173847:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00100000:2.0:1784949237.173848:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 808 00000100:00000001:1.0:1784949237.173849:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306176555 00000400:00000200:0.0:1784949237.173854:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949237.173862:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306176555 00000100:00000001:3.0:1784949237.173865:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000040:2.0:1784949237.173866:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:0.0:1784949237.173869:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949237.173873:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306176555 00000100:00000001:2.0:1784949237.173880:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466826832 : -115544242724784 : ffff96e9c274e050) 00000100:00000001:3.0:1784949237.173883:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949237.173884:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306176555 00000400:00000200:0.0:1784949237.173885:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784949237.173898:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306176555 00000100:00000001:3.0:1784949237.173900:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949237.173901:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949237.173946:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3792:3792:240:4294967295] 192.168.202.137@tcp LPNI seq info [2427:2427:7:4294967295] 00000100:00000040:2.0:1784949237.173953:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ab8619ba x1871651289683712/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:143/0 lens 224/0 e 0 to 0 dl 1784949248 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.173954:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949237.173966:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.173968:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949237.173978:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.173984:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949237.174013:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949237.174014:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949237.174028:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949237.174035:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:0.0:1784949237.174042:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949237.174043:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949237.174048:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949237.174052:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949237.174055:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949237.174056:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949237.174057:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002a753d9d. 00000100:00000001:1.0:1784949237.174058:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949237.174070:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949237.174070:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000800:00000200:0.0:1784949237.174081:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949237.174084:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1784949237.174095:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000020:00000010:2.0:1784949237.174096:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000800:00000200:0.0:1784949237.174104:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002a753d9d type 1, nob 320 niov 1 nkiov 1 00000100:00100000:2.0:1784949237.174114:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ab8619ba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289683712:12345-0@lo:400: 00000100:00000001:0.0:1784949237.174123:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949237.174126:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289683712 00000100:00000040:0.0:1784949237.174131:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2227 usecs (3 jiffies) 00000020:00000001:2.0:1784949237.174136:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949237.174138:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306176517 00000100:00000001:0.0:1784949237.174147:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306176555 00000020:00000001:2.0:1784949237.174151:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949237.174152:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306176517 00000100:00000001:0.0:1784949237.174161:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306176555 00000020:00000001:2.0:1784949237.174161:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949237.174166:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306176517 00000100:00000001:0.0:1784949237.174174:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306176555 00000020:00000001:2.0:1784949237.174174:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949237.174182:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306176517 00000020:00000001:2.0:1784949237.174185:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784949237.174187:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306176555 00000800:00000200:1.0:1784949237.174192:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949237.174194:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1784949237.174201:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000010:1.0:1784949237.174202:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008d5370ec. 00000100:00000001:0.0:1784949237.174203:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949237.174211:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949237.174216:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949237.174218:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949237.174221:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784949237.174224:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949237.174230:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:1.0:1784949237.174235:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949237.174240:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949237.174243:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000020:00000001:2.0:1784949237.174249:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000010:1.0:1784949237.174250:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:2.0:1784949237.174260:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949237.174261:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:0.0:1784949237.174270:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784949237.174272:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000200:1.0:1784949237.174275:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949237.174277:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000010:2.0:1784949237.174285:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c3dbeb98. 00000800:00000010:3.0:1784949237.174287:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002a753d9d. 00000800:00000001:0.0:1784949237.174290:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949237.174301:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784949237.174301:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949237.174301:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949237.174310:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949237.174315:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949237.174317:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949237.174327:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949237.174329:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949237.174330:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949262 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949237.174334:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000040:2.0:1784949237.174345:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289683712 00000400:00000200:3.0:1784949237.174346:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000414ed034 00000400:00000010:3.0:1784949237.174353:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000414ed034. 00010000:00000001:2.0:1784949237.174363:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:0.0:1784949237.174366:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643d780 00000100:00000001:3.0:1784949237.174366:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949237.174380:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:2.0:1784949237.174381:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ab8619ba x1871651289683712/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:143/0 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949237.174382:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.174387:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000100:0.0:1784949237.174395:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289683840 offset 0 length 520: 4 00000800:00000001:1.0:1784949237.174402:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949237.174413:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 0000000023ed5eac is in inconsistent state, don't perform health checking (-2, 0) 00000800:00000001:1.0:1784949237.174417:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949237.174427:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:1.0:1784949237.174430:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949237.174439:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.174447:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:0.0:1784949237.174453:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784949237.174457:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:3.0:1784949237.174463:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949237.174464:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000001:2.0:1784949237.174466:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949237.174480:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949237.174483:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=109 v=5 (1 1 1 1) 00000800:00000001:0.0:1784949237.174495:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949237.174497:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d800 00000100:00000001:3.0:1784949237.174498:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949237.174502:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:0.0:1784949237.174508:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.174509:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1784949237.174517:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000400:00000200:1.0:1784949237.174521:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 176656 00000100:00000001:2.0:1784949237.174530:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:1.0:1784949237.174536:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784949237.174539:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949237.174546:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949237.174553:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:1.0:1784949237.174555:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949237.174565:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949237.174566:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:1.0:1784949237.174577:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:2.0:1784949237.174579:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000001:1.0:1784949237.174584:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949237.174589:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:2.0:1784949237.174591:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289683712, offset 224 00000100:00000010:1.0:1784949237.174596:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000086a153f0. 00000100:00000040:1.0:1784949237.174603:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000086a153f0 x1871651289683968 msgsize 224 00000400:00000200:2.0:1784949237.174611:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:1.0:1784949237.174617:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784949237.174634:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949237.174643:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949237.174654:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d700 00000800:00000001:1.0:1784949237.174655:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949237.174663:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949237.174672:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9365 [1] + 224 00000400:00000200:2.0:1784949237.174687:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949237.174695:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949237.174712:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949237.174713:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1784949237.174719:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289683968 02000000:00000001:3.0:1784949237.174724:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949237.174728:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784949237.174729:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949237.174732:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949237.174740:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949237.174748:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289683968 00000020:00000001:3.0:1784949237.174755:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949237.174758:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949237.174764:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949237.174769:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949237.174775:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949237.174780:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:2.0:1784949237.174781:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949237.174787:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949237.174796:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000094fcdc7f. 00000100:00000001:3.0:1784949237.174805:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949237.174815:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000094fcdc7f. 00000020:00000010:3.0:1784949237.174825:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000035b54a67. 00000020:00000010:3.0:1784949237.174831:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000d656743e. 00000100:00000001:2.0:1784949237.174837:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784949237.174839:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000200:2.0:1784949237.174854:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000100:00000040:3.0:1784949237.174857:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:2.0:1784949237.174863:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:3.0:1784949237.174867:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949237.174873:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949237.174874:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949237.174878:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949237.174882:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949237.174890:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949237.174894:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.174897:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949237.174939:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949237.174946:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949237.174949:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784949237.174953:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1784949237.174960:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:0.0:1784949237.174966:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1784949237.174967:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.174968:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:0.0:1784949237.174973:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949237.174983:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000061b5c7fd. 00000020:00000001:2.0:1784949237.174983:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949237.175027:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ab8619ba x1871651289683712/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:143/0 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949237.175028:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784949237.175030:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000400:00000010:0.0:1784949237.175037:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000100:00000001:3.0:1784949237.175037:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949237.175050:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1784949237.175060:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 784 00000100:00000200:0.0:1784949237.175065:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949237.175065:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949237.175070:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00100000:2.0:1784949237.175072:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ab8619ba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289683712:12345-0@lo:400: Request processed in 1011us (1833us total) trans 0 rc 0/0 00000100:00000001:1.0:1784949237.175081:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949237.175089:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637807360 : -115544071744256 : ffff96e9cca5d500) 00000100:00000001:1.0:1784949237.175090:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949237.175105:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949237.175115:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000086a153f0 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:143/0 lens 224/0 e 0 to 0 dl 1784949248 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949237.175119:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 808 00000100:00000040:2.0:1784949237.175143:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:0.0:1784949237.175148:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949237.175156:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949237.175163:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949237.175174:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949237.175176:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000738a21fc. 00000100:00000001:0.0:1784949237.175178:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949237.175183:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949237.175185:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000094fcdc7f. 00000100:00000001:0.0:1784949237.175191:0:337119:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:2.0:1784949237.175191:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784949237.175193:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949237.175197:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949237.175205:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:2.0:1784949237.175205:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000098dd29a0. 00000020:00000010:3.0:1784949237.175206:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000094fcdc7f. 00000100:00000001:0.0:1784949237.175212:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949237.175220:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949237.175223:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:0.0:1784949237.175226:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=109 v=5 (5 5 5 5) 00000100:00100000:3.0:1784949237.175231:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000086a153f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289683968:12345-192.168.202.137@tcp:400: 02000000:00000001:2.0:1784949237.175238:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:0.0:1784949237.175241:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=109 v=5 (1 1 1 1) 02000000:00000010:2.0:1784949237.175247:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c3dbeb98. 00000100:00000200:3.0:1784949237.175250:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289683968 00000100:00000001:0.0:1784949237.175251:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949237.175259:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949237.175260:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:2.0:1784949237.175261:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:0.0:1784949237.175264:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784949237.175270:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949237.175271:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949237.175271:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949237.175275:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949237.175280:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949237.175289:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949237.175290:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000200:2.0:1784949237.175291:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:3.0:1784949237.175299:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000010:2.0:1784949237.175307:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005cc7db3b. 00000020:00000001:3.0:1784949237.175315:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949237.175324:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949237.175324:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005456a5d6. 00000020:00000001:3.0:1784949237.175326:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949237.175337:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1784949237.175338:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784949237.175343:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949237.175345:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949237.175351:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949237.175357:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949237.175359:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949237.175368:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00100000:0.0:1784949237.175374:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289683712:0@lo:400: 02000000:00000001:3.0:1784949237.175375:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949237.175394:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784949237.175395:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000cc32100. 00000100:00000001:0.0:1784949237.175399:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784949237.175402:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949237.175408:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003bf8e8ad x1871651289683712/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949237.175410:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949237.175419:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949237.175428:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289683968 00010000:00000001:3.0:1784949237.175437:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949237.175441:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000200:3.0:1784949237.175448:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000086a153f0 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:143/0 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949237.175451:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949237.175459:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 00010000:00000001:3.0:1784949237.175471:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949237.175474:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949237.175476:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00001000:3.0:1784949237.175478:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=104 v=5 (1 1 1 1) 00000020:00000001:0.0:1784949237.175485:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784949237.175486:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:0.0:1784949237.175493:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000040:3.0:1784949237.175495:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000001:0.0:1784949237.175508:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784949237.175508:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949237.175518:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000010:0.0:1784949237.175519:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:3.0:1784949237.175521:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.175530:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949237.175532:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:3.0:1784949237.175536:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:0.0:1784949237.175539:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000010:3.0:1784949237.175542:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000100:00000010:0.0:1784949237.175546:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003bf8e8ad. 00000100:00000200:3.0:1784949237.175548:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289683968, offset 224 00000100:00000001:0.0:1784949237.175561:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000400:00000200:3.0:1784949237.175562:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949237.175571:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949237.175581:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949237.175581:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949237.175593:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3793:3793:241:4294967295] 192.168.202.137@tcp LPNI seq info [2428:2428:8:4294967295] 00000100:00000001:0.0:1784949237.175596:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949237.175603:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949237.175615:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949237.175617:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949237.175624:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949237.175631:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949237.175638:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784949237.175644:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949237.175652:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949237.175656:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949237.175665:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949237.175671:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949237.175675:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949237.175677:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949237.175680:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949237.175685:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086a153f0 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:143/0 lens 224/224 e 0 to 0 dl 1784949248 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949237.175697:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000086a153f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289683968:12345-192.168.202.137@tcp:400: Request processed in 523us (1092us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949237.175709:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 784 00000100:00000040:3.0:1784949237.175713:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949237.175716:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949237.175718:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949237.175722:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000d656743e. 00000020:00000010:3.0:1784949237.175726:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949237.175731:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000035b54a67. 00000020:00000040:3.0:1784949237.175738:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949237.175742:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949237.175771:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949237.175777:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:0.0:1784949237.175785:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949237.175792:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949237.175797:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:0.0:1784949237.175799:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000100:00000001:0.0:1784949237.175803:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949237.175805:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949237.175812:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949237.175820:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949237.175822:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949237.175825:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949237.175830:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949237.175843:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d800 00000400:00000200:0.0:1784949237.175851:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9375 [1] + 224 00000800:00000001:0.0:1784949237.175858:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949237.175864:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949237.175867:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949237.175871:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949237.175876:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949237.175882:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949237.175894:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949237.175948:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949237.175955:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949237.175960:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.176021:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949237.176037:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949237.176042:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949237.176044:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949237.176047:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000001881934. 00000400:00000200:3.0:1784949237.176053:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f594d710 00000400:00000010:3.0:1784949237.176055:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f594d710. 00000100:00000001:3.0:1784949237.176058:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949237.176065:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949237.176076:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949237.176085:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949237.176086:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.176089:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949237.176091:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949237.176093:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949237.176095:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949237.176097:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949237.176103:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=104 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949237.176106:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=104 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949237.176109:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949237.176111:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949237.176113:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949237.176118:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949237.176126:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.176130:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949237.176132:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949237.176134:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949237.176136:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949237.176139:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949237.176147:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949237.176149:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949237.176150:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949237.176154:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949237.176163:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289683968:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949237.176168:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.176170:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949237.176172:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000aedb1960 x1871651289683968/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949237.176179:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949237.176181:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949237.176183:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 02000000:00000001:3.0:1784949237.176189:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949237.176190:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949237.176191:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949237.176194:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949237.176195:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005a1c05a3. 02000000:00000001:3.0:1784949237.176198:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949237.176200:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949237.176201:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000aedb1960. 00000100:00000001:3.0:1784949237.176205:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949237.176206:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.176209:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949237.176213:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949237.176214:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949237.176217:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949237.176219:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949237.216054:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306176555, at 4306176556 00000100:00000010:3.0:1784949237.216069:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000040:0.0:1784949237.216080:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306176555, at 4306176556 00000100:00000040:1.0:1784949237.216087:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306176555, at 4306176556 00000100:00000040:2.0:1784949237.216092:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306176555, at 4306176556 00000100:00000010:0.0:1784949237.216100:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:1.0:1784949237.216103:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000486261c. 00000100:00000010:2.0:1784949237.216113:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003e9ea21c. 00010000:00000001:2.0:1784949238.004064:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949238.004072:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949238.004079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949238.004081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949238.004216:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949238.004222:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949238.004227:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949239.028072:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949239.028081:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949239.028089:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949239.028091:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949239.028202:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949239.028209:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949239.028214:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949240.052018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949240.052025:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949240.052031:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949240.052033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949240.052105:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949240.052110:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949240.052113:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949241.076030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949241.076036:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949241.076042:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949241.076045:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949241.076047:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949241.076049:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949241.076052:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949241.076054:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949241.076055:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949241.076056:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949241.076114:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949241.076118:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949241.076121:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949242.100067:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949242.100076:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949242.100085:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949242.100087:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949242.100196:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949242.100203:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949242.100208:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949242.292062:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949242.292078:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949242.292081:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949242.292088:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949242.292104:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949242.292108:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.292113:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949242.292119:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949242.292132:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949242.292145:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003bf8e8ad. 00000020:00000040:0.0:1784949242.292157:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949242.292166:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949242.292172:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949242.292180:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949242.292193:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949242.292204:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949242.292219:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949242.292239:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949242.292243:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949242.292251:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf8e8ad] to pc [ptlrpcd_00_03+3] req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949242.292310:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949242.292315:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949242.292321:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949242.292325:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949242.292331:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949242.292337:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004614f9cb. 00000020:00000040:0.0:1784949242.292342:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949242.292347:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949242.292351:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949242.292355:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.292360:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949242.292364:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000662152ce. 00000100:00100000:1.0:1784949242.292368:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:0.0:1784949242.292373:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.292374:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949242.292381:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949242.292389:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949242.292390:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949242.292396:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949242.292397:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949242.292407:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949242.292413:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949242.292419:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949242.292432:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949242.292482:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949242.292486:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949242.292487:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949242.292493:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949242.292496:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949242.292502:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949242.292510:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949242.292511:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004614f9cb] to pc [ptlrpcd_00_00+0] req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949242.292521:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289684096:0@lo:400: 00000100:00000001:1.0:1784949242.292536:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949242.292546:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949242.292551:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949242.292556:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949242.292563:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 02000000:00000001:1.0:1784949242.292567:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949242.292575:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000eed58f44. 00000100:00000001:0.0:1784949242.292578:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949242.292585:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000040:0.0:1784949242.292590:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11519) 00000100:00000200:1.0:1784949242.292597:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684096, portal 25 00000100:00000001:3.0:1784949242.292603:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949242.292605:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949242.292611:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469562752 : -115544239988864 : ffff96e9c29e9f80) 00000100:00000001:3.0:1784949242.292612:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949242.292621:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949242.292627:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949242.292637:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949242.292645:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949242.292648:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949242.292656:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949242.292665:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949242.292673:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000100:00000001:3.0:1784949242.292677:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:1.0:1784949242.292681:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289684096, offset 0 00000100:00000001:3.0:1784949242.292683:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949242.292693:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784949242.292699:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784949242.292700:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949242.292709:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949242.292715:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289684224:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949242.292721:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949242.292725:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1784949242.292729:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:3.0:1784949242.292732:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949242.292738:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949242.292745:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dbce179e. 00000400:00000200:1.0:1784949242.292751:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d880 02000000:00000001:3.0:1784949242.292753:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949242.292765:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000af4b2538. 00000400:00000200:1.0:1784949242.292767:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x896d [2] + 0 00000400:00000010:1.0:1784949242.292777:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000400:00000200:1.0:1784949242.292784:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000096ea68f5 00000400:00000010:3.0:1784949242.292795:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000400:00000200:1.0:1784949242.292798:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:3.0:1784949242.292804:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684224, portal 10 00000100:00000001:3.0:1784949242.292814:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949242.292815:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949242.292817:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469554688 : -115544239996928 : ffff96e9c29e8000) 00000400:00000200:1.0:1784949242.292835:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000096ea68f5 00000100:00000040:3.0:1784949242.292836:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949242.292845:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000096ea68f5. 00000100:00000001:3.0:1784949242.292859:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949242.292863:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784949242.292865:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:1.0:1784949242.292872:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:3.0:1784949242.292873:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003dcd3ddc. 00000100:00000200:3.0:1784949242.292882:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289684224, offset 0 00000100:00000040:1.0:1784949242.292887:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000097ccad2 x1871651289684096 msgsize 224 00000400:00000200:3.0:1784949242.292895:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949242.292979:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949242.292988:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:1.0:1784949242.292999:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784949242.293000:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:1.0:1784949242.293012:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949242.293014:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949242.293024:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949242.293029:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3794:3794:241:4294967295] 192.168.202.137@tcp LPNI seq info [2429:2429:8:4294967295] 00000100:00000001:1.0:1784949242.293039:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949242.293049:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949242.293057:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949242.293064:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000400:00000200:1.0:1784949242.293066:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000400:00000010:1.0:1784949242.293072:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000100:00000001:1.0:1784949242.293079:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784949242.293080:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:1.0:1784949242.293093:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1784949242.293105:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784949242.293106:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949242.293118:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684096 00000800:00000200:3.0:1784949242.293119:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949242.293125:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949242.293130:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:1.0:1784949242.293137:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949242.293139:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949242.293148:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293157:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.293161:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949242.293163:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949242.293167:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2228 usecs (3 jiffies) 00000100:00000001:1.0:1784949242.293168:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1784949242.293174:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684096 00000100:00000001:3.0:1784949242.293177:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306181636 00000100:00000001:1.0:1784949242.293179:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949242.293184:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949242.293192:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949242.293194:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306181675 00000100:00000010:3.0:1784949242.293195:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000020:00000001:0.0:1784949242.293198:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949242.293209:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949242.293212:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000001:0.0:1784949242.293219:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949242.293227:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306181636 00000020:00000001:0.0:1784949242.293228:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949242.293233:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306181636 00000100:00000010:3.0:1784949242.293238:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:0.0:1784949242.293243:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949242.293248:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306181675 00000100:00000001:3.0:1784949242.293249:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000010:0.0:1784949242.293255:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:3.0:1784949242.293266:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306181636 00000100:00000001:0.0:1784949242.293267:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949242.293269:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306181636 00000100:00000010:3.0:1784949242.293281:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000020:00000010:0.0:1784949242.293283:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00000001:1.0:1784949242.293285:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306181675 00000100:00000001:3.0:1784949242.293293:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949242.293304:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306181636 00000020:00000010:0.0:1784949242.293305:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000001:1.0:1784949242.293306:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306181636 00000100:00000010:3.0:1784949242.293313:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003e9ea21c. 00000100:00000001:1.0:1784949242.293319:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306181675 00000100:00000001:3.0:1784949242.293322:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000010:0.0:1784949242.293323:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a9a427e3. 00000100:00000001:3.0:1784949242.293334:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949242.293337:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306181636 00000020:00000010:0.0:1784949242.293338:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000006bcd339e. 00000100:00000001:3.0:1784949242.293346:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.293350:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949242.293352:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.293358:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293360:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949242.293362:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949242.293366:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293373:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949242.293375:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949242.293376:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293379:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949242.293385:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949242.293388:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949242.293392:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293394:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949242.293403:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949242.293408:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949242.293409:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949242.293412:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949242.293424:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784949242.293428:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949242.293432:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000800:00000001:1.0:1784949242.293434:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949242.293442:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949242.293445:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949242.293445:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949242.293466:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784949242.293469:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949242.293474:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784949242.293477:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949242.293478:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949242.293487:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949242.293487:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003dcd3ddc 00000100:00100000:3.0:1784949242.293493:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 809 00000400:00000010:2.0:1784949242.293497:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003dcd3ddc. 00000100:00000040:3.0:1784949242.293502:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949242.293509:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1784949242.293510:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643d900 00000100:00000001:0.0:1784949242.293511:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293512:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466820688 : -115544242730928 : ffff96e9c274c850) 00000100:00000001:0.0:1784949242.293526:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949242.293527:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949242.293536:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000097ccad2 x1871651289684096/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:148/0 lens 224/0 e 0 to 0 dl 1784949253 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949242.293543:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 176880 00000800:00000001:1.0:1784949242.293570:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949242.293586:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949242.293589:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949242.293592:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784949242.293595:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949242.293600:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000586a9cb7. 00000100:00000040:2.0:1784949242.293602:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949242.293611:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949242.293615:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949242.293624:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000586a9cb7. 00000100:00000001:1.0:1784949242.293633:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949242.293640:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784949242.293646:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949242.293647:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000097ccad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289684096:12345-0@lo:400: 00000100:00000010:1.0:1784949242.293652:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001dee7260. 00000100:00000001:2.0:1784949242.293656:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:3.0:1784949242.293667:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684096 00000100:00000040:1.0:1784949242.293668:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000001dee7260 x1871651289684224 msgsize 224 00000020:00000001:3.0:1784949242.293676:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:1.0:1784949242.293686:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:3.0:1784949242.293687:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949242.293694:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949242.293700:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949242.293705:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949242.293709:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949242.293711:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949242.293713:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:3.0:1784949242.293715:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949242.293721:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949242.293725:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949242.293727:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949242.293728:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949242.293736:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000001:1.0:1784949242.293737:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949242.293742:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949242.293750:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007afc0d3b. 02000000:00000001:3.0:1784949242.293754:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293757:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949242.293761:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949242.293765:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684096 00010000:00000001:3.0:1784949242.293769:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949242.293777:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000097ccad2 x1871651289684096/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:148/0 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949242.293785:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:3.0:1784949242.293789:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949242.293794:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949242.293795:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684224 00000100:00001000:3.0:1784949242.293800:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=114 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949242.293803:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949242.293810:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949242.293813:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949242.293816:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949242.293821:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 02000000:00000001:0.0:1784949242.293828:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.293829:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949242.293841:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00100000:0.0:1784949242.293842:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684224 02000000:00000001:3.0:1784949242.293844:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949242.293851:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949242.293853:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1784949242.293858:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000040:3.0:1784949242.293861:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:0.0:1784949242.293867:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:3.0:1784949242.293869:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000020:00000040:0.0:1784949242.293874:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000200:3.0:1784949242.293876:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289684096, offset 224 00000020:00000001:0.0:1784949242.293888:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:3.0:1784949242.293894:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:0.0:1784949242.293902:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949242.293975:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949242.293981:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784949242.293985:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:0.0:1784949242.293994:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949242.293996:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d880 00000020:00000010:0.0:1784949242.294005:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000400:00000200:3.0:1784949242.294012:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9389 [1] + 224 00000020:00000010:0.0:1784949242.294021:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000400:00000200:3.0:1784949242.294024:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949242.294033:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:0.0:1784949242.294034:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006459e3c3. 00000100:00000001:3.0:1784949242.294039:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1784949242.294046:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002ffa19fa. 00000100:00000200:3.0:1784949242.294047:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949242.294067:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:3.0:1784949242.294068:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949242.294079:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949242.294086:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949242.294093:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949242.294102:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949242.294103:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949242.294112:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000200:3.0:1784949242.294116:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00000100:00000001:0.0:1784949242.294117:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949242.294122:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000100:00000001:3.0:1784949242.294129:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949242.294132:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949242.294137:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949242.294143:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784949242.294144:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949242.294148:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949242.294154:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.294157:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949242.294158:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949242.294166:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949242.294170:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949242.294171:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000097ccad2 x1871651289684096/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:148/0 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949242.294178:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784949242.294180:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000001:1.0:1784949242.294187:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949242.294194:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000eed58f44. 00000100:00100000:0.0:1784949242.294203:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 785 00000100:00100000:3.0:1784949242.294205:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000097ccad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289684096:12345-0@lo:400: Request processed in 607us (1312us total) trans 0 rc 0/0 00000400:00000200:1.0:1784949242.294207:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949242.294217:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000040:0.0:1784949242.294220:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949242.294229:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949242.294231:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637811840 : -115544071739776 : ffff96e9cca5e680) 00000100:00100000:3.0:1784949242.294232:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 809 00000100:00000200:1.0:1784949242.294243:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949242.294244:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949242.294253:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000040:0.0:1784949242.294255:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001dee7260 x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:148/0 lens 224/0 e 0 to 0 dl 1784949253 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949242.294260:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949242.294271:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a9a427e3. 00000020:00000010:3.0:1784949242.294285:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000006bcd339e. 00000020:00000010:3.0:1784949242.294298:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 00000100:00000200:1.0:1784949242.294303:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949242.294313:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000040:3.0:1784949242.294314:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949242.294322:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1784949242.294326:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000010:0.0:1784949242.294327:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:1.0:1784949242.294328:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784949242.294334:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007afc0d3b. 00000100:00000001:1.0:1784949242.294340:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:0.0:1784949242.294342:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949242.294344:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:1.0:1784949242.294347:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949242.294350:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949242.294353:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:0.0:1784949242.294356:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00000001:1.0:1784949242.294358:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949242.294359:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:1.0:1784949242.294370:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949242.294374:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d97ec0aa. 00000100:00001000:1.0:1784949242.294381:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=114 v=5 (5 5 5 5) 00000100:00100000:0.0:1784949242.294385:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001dee7260 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289684224:12345-192.168.202.137@tcp:400: 00000400:00000010:3.0:1784949242.294388:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000054d1fd97. 00000100:00001000:1.0:1784949242.294390:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=114 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949242.294404:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000200:0.0:1784949242.294407:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684224 00000100:00000001:1.0:1784949242.294409:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949242.294417:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00010000:00000001:1.0:1784949242.294418:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949242.294426:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949242.294428:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949242.294431:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949242.294436:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949242.294442:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949242.294446:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949242.294452:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784949242.294466:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949242.294468:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949242.294476:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1784949242.294477:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949242.294480:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949242.294484:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784949242.294490:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949242.294493:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949242.294501:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949242.294511:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:1.0:1784949242.294516:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289684096:0@lo:400: 00000100:00000001:0.0:1784949242.294520:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949242.294524:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949242.294528:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949242.294529:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949242.294536:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003bf8e8ad x1871651289684096/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949242.294540:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000994a91fc. 02000000:00000001:0.0:1784949242.294550:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.294554:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784949242.294557:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949242.294560:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:0.0:1784949242.294566:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949242.294567:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 00010000:00000040:0.0:1784949242.294579:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684224 02000000:00000001:1.0:1784949242.294580:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949242.294586:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00010000:00000001:0.0:1784949242.294594:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000040:1.0:1784949242.294594:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949242.294599:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949242.294605:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 00010000:00000200:0.0:1784949242.294610:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001dee7260 x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:148/0 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949242.294613:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949242.294619:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949242.294624:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003bf8e8ad. 00000100:00000001:1.0:1784949242.294636:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00010000:00000001:0.0:1784949242.294638:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949242.294640:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784949242.294646:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.294647:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784949242.294658:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=109 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949242.294663:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949242.294667:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949242.294668:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949242.294675:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949242.294679:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949242.294681:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949242.294688:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949242.294694:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949242.294696:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949242.294699:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949242.294706:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949242.294711:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d22072f2. 00000100:00000200:0.0:1784949242.294715:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289684224, offset 224 00000400:00000200:0.0:1784949242.294721:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949242.294731:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949242.294738:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3795:3795:241:4294967295] 192.168.202.137@tcp LPNI seq info [2430:2430:8:4294967295] 00000400:00000200:0.0:1784949242.294749:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949242.294756:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949242.294760:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000820f02b3. 00000800:00000200:0.0:1784949242.294765:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949242.294772:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949242.294776:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000820f02b3 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949242.294796:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949242.294800:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949242.294803:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949242.294805:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949242.294808:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949242.294812:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001dee7260 x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:148/0 lens 224/224 e 0 to 0 dl 1784949253 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949242.294824:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001dee7260 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289684224:12345-192.168.202.137@tcp:400: Request processed in 512us (1156us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949242.294834:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 785 00000100:00000040:0.0:1784949242.294838:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949242.294841:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949242.294843:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949242.294846:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006459e3c3. 00000020:00000010:0.0:1784949242.294850:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002ffa19fa. 00000020:00000010:0.0:1784949242.294854:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000020:00000040:0.0:1784949242.294859:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949242.294863:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949242.295034:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949242.295040:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000820f02b3. 00000400:00000200:2.0:1784949242.295048:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949242.295056:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949242.295062:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d22072f2 00000400:00000010:2.0:1784949242.295065:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d22072f2. 00000100:00000001:2.0:1784949242.295070:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949242.295073:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949242.295077:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949242.295091:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949242.295096:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949242.295101:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949242.295110:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949242.295121:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643d900 00000400:00000200:1.0:1784949242.295127:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9391 [1] + 224 00000800:00000001:1.0:1784949242.295133:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949242.295138:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949242.295141:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949242.295145:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949242.295150:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949242.295155:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949242.295166:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949242.295188:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949242.295195:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949242.295198:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949242.295257:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949242.295264:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949242.295271:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949242.295273:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949242.295276:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000af4b2538. 00000400:00000200:3.0:1784949242.295285:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:3.0:1784949242.295288:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000100:00000001:3.0:1784949242.295292:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949242.295300:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949242.295313:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949242.295323:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949242.295325:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949242.295328:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949242.295330:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949242.295332:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949242.295334:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949242.295337:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949242.295343:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=109 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949242.295347:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=109 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949242.295350:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949242.295352:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949242.295354:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949242.295358:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949242.295368:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.295371:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949242.295373:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949242.295375:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949242.295377:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949242.295380:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949242.295388:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949242.295392:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949242.295394:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949242.295397:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949242.295407:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289684224:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949242.295411:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949242.295414:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949242.295416:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004614f9cb x1871651289684224/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949242.295423:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949242.295425:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949242.295427:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dbce179e. 02000000:00000001:3.0:1784949242.295434:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949242.295436:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949242.295437:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949242.295440:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949242.295442:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000662152ce. 02000000:00000001:3.0:1784949242.295445:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949242.295447:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949242.295448:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004614f9cb. 00000100:00000001:3.0:1784949242.295453:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949242.295454:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949242.295456:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949242.295461:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949242.295463:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949242.295466:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949242.295468:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949242.336060:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306181675, at 4306181676 00000100:00000040:3.0:1784949242.336067:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306181675, at 4306181676 00000100:00000040:0.0:1784949242.336072:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306181675, at 4306181676 00000100:00000010:1.0:1784949242.336079:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000010:3.0:1784949242.336083:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003e9ea21c. 00000100:00000010:0.0:1784949242.336093:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000040:2.0:1784949242.336102:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306181675, at 4306181676 00000100:00000010:2.0:1784949242.336117:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00010000:00000001:2.0:1784949243.124065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949243.124078:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949243.124089:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949243.124093:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949243.124185:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949243.124192:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949243.124197:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949243.316158:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949243.316176:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949243.316180:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949243.316188:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949243.316208:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949243.316213:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949243.316218:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949244.148077:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949244.148089:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949244.148100:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949244.148103:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949244.148192:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949244.148198:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949244.148203:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949244.340076:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949244.340095:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949244.340098:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949244.340106:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949244.340110:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949244.340115:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949244.340119:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949245.172089:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949245.172099:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949245.172107:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949245.172109:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949245.172202:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949245.172219:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949245.172226:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949245.364037:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949245.364060:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949245.364065:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949245.364075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949245.364080:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949245.364086:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949245.364091:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949246.196057:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949246.196066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949246.196074:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949246.196077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949246.196194:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949246.196201:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949246.196206:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949246.388147:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949246.388168:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949246.388173:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949246.388182:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949246.388187:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949246.388193:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949246.388199:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949247.220065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949247.220075:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949247.220086:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949247.220088:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949247.220166:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949247.220172:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949247.220176:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949247.412041:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949247.412058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949247.412061:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949247.412068:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949247.412073:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949247.412077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.412081:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949247.412084:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949247.412094:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949247.412106:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006e540bde. 00000020:00000040:3.0:1784949247.412115:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949247.412122:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949247.412127:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949247.412133:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949247.412143:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:3.0:1784949247.412153:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949247.412166:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949247.412182:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949247.412185:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949247.412191:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e540bde] to pc [ptlrpcd_00_01+1] req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949247.412225:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949247.412229:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949247.412233:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949247.412236:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949247.412241:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949247.412244:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000040:3.0:1784949247.412248:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949247.412251:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949247.412253:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949247.412256:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.412262:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784949247.412263:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:2.0:1784949247.412270:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949247.412272:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.412276:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949247.412280:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949247.412284:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1784949247.412289:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949247.412290:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949247.412333:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949247.412337:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949247.412339:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.412344:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949247.412351:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_02+2] req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949247.412354:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949247.412361:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949247.412371:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949247.412379:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289684352:0@lo:400: 00000100:00000001:2.0:1784949247.412385:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949247.412392:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949247.412394:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949247.412395:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949247.412404:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1784949247.412407:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11524) 02000000:00000010:2.0:1784949247.412412:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000098dd29a0. 02000000:00000001:2.0:1784949247.412419:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949247.412428:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006138467e. 00000100:00000001:0.0:1784949247.412434:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784949247.412440:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00100000:0.0:1784949247.412443:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:0.0:1784949247.412449:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949247.412453:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684352, portal 25 00000100:00000001:0.0:1784949247.412459:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949247.412460:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949247.412466:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949247.412467:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554957056 : -115544154594560 : ffff96e9c7b5a300) 00000100:00000001:1.0:1784949247.412474:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949247.412477:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949247.412483:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949247.412485:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949247.412487:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949247.412492:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949247.412505:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949247.412507:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949247.412515:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949247.412523:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000100:00000200:2.0:1784949247.412531:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289684352, offset 0 00000100:00000001:0.0:1784949247.412544:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949247.412549:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949247.412551:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:0.0:1784949247.412557:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949247.412562:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949247.412565:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949247.412571:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289684480:192.168.202.137@tcp:400: 00000400:00000200:2.0:1784949247.412580:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949247.412584:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949247.412590:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949247.412593:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949247.412597:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784949247.412603:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643d980 02000000:00000010:0.0:1784949247.412607:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949247.412615:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949247.412616:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x89fd [2] + 0 00000400:00000010:2.0:1784949247.412626:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000075224c9. 00000400:00000010:0.0:1784949247.412627:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000948ad730. 00000400:00000200:2.0:1784949247.412634:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000088cde038 00000400:00000200:2.0:1784949247.412647:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:0.0:1784949247.412649:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000400:00000200:2.0:1784949247.412658:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:0.0:1784949247.412661:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684480, portal 10 00000400:00000200:2.0:1784949247.412667:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000088cde038 00000100:00000001:0.0:1784949247.412668:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949247.412673:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000400:00000010:2.0:1784949247.412675:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000088cde038. 00000100:00000001:2.0:1784949247.412687:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:0.0:1784949247.412688:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949247.412694:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949247.412704:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000008db39772 x1871651289684352 msgsize 224 00000100:00000001:0.0:1784949247.412718:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1784949247.412722:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949247.412732:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784949247.412733:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000010:0.0:1784949247.412738:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000993e9fad. 00000100:00000200:0.0:1784949247.412744:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289684480, offset 0 00000400:00000200:0.0:1784949247.412751:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949247.412758:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949247.412767:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949247.412775:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949247.412780:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949247.412786:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949247.412791:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949247.412797:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3796:3796:241:4294967295] 192.168.202.137@tcp LPNI seq info [2431:2431:8:4294967295] 00000400:00000200:0.0:1784949247.412815:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949247.412824:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949247.412830:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000800:00000010:0.0:1784949247.412832:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000662152ce. 00000400:00000010:2.0:1784949247.412836:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000100:00000001:1.0:1784949247.412838:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949247.412846:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:0.0:1784949247.412849:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:1.0:1784949247.412850:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684352 00000100:00000200:2.0:1784949247.412859:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949247.412861:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949247.412870:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:0.0:1784949247.412879:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949247.412880:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949247.412894:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000662152ce type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949247.412895:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784949247.412898:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949247.412958:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949247.412985:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949247.412988:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.412990:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1784949247.413002:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2231 usecs (3 jiffies) 00000100:00100000:1.0:1784949247.413003:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684352 00000100:00000001:2.0:1784949247.413006:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949247.413014:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306186756 00000020:00000001:1.0:1784949247.413016:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949247.413017:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306186795 00000020:00000040:1.0:1784949247.413026:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:0.0:1784949247.413028:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000020:00000001:1.0:1784949247.413035:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949247.413044:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000020:00000040:1.0:1784949247.413044:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949247.413057:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306186756 00000020:00000001:1.0:1784949247.413057:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949247.413063:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306186756 00000020:00000001:1.0:1784949247.413068:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:0.0:1784949247.413070:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:2.0:1784949247.413078:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306186795 00000100:00000001:0.0:1784949247.413083:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949247.413084:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949247.413097:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306186756 00000020:00000010:1.0:1784949247.413098:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f7efedb6. 00000100:00000001:2.0:1784949247.413100:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306186756 00000100:00000010:0.0:1784949247.413106:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:1.0:1784949247.413114:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949247.413116:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949247.413118:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306186795 00000100:00000001:0.0:1784949247.413128:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306186756 00000020:00000010:1.0:1784949247.413131:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f7efedb6. 00000100:00000001:2.0:1784949247.413133:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306186756 00000100:00000010:0.0:1784949247.413138:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:0.0:1784949247.413147:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949247.413148:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306186795 00000020:00000010:1.0:1784949247.413154:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000b241612. 00000100:00000001:0.0:1784949247.413155:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949247.413159:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306186756 00000100:00000001:0.0:1784949247.413168:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949247.413173:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000052e9381a. 00000100:00000001:2.0:1784949247.413174:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949247.413175:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949247.413176:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949247.413189:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000662152ce. 00000100:00000001:2.0:1784949247.413190:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949247.413192:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949247.413193:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000400:00000200:3.0:1784949247.413201:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949247.413203:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.413210:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949247.413219:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949247.413221:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949247.413222:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949247.413227:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:3.0:1784949247.413229:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000993e9fad 00000100:00000001:1.0:1784949247.413233:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000800:00000001:0.0:1784949247.413239:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949247.413240:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949247.413240:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000993e9fad. 00000100:00000001:1.0:1784949247.413241:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949247.413248:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949247.413252:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949247.413253:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949247.413254:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949247.413259:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949247.413265:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949247.413266:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949247.413268:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949247.413271:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949247.413279:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949247.413302:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949247.413303:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949247.413307:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949247.413315:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949247.413327:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643da00 00000100:00000001:1.0:1784949247.413336:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949247.413346:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:0.0:1784949247.413349:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 177104 00000100:00000001:3.0:1784949247.413351:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949247.413361:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1784949247.413364:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 810 00000800:00000001:0.0:1784949247.413371:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949247.413375:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000800:00000001:0.0:1784949247.413385:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949247.413385:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466830928 : -115544242720688 : ffff96e9c274f050) 00000400:00000200:0.0:1784949247.413396:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784949247.413406:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000008db39772 x1871651289684352/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:153/0 lens 224/0 e 0 to 0 dl 1784949258 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949247.413410:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949247.413427:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949247.413432:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784949247.413440:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:0.0:1784949247.413441:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000de534d9b. 00000100:00000040:0.0:1784949247.413448:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000de534d9b x1871651289684480 msgsize 224 00000020:00000010:1.0:1784949247.413451:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f7efedb6. 00000100:00100000:0.0:1784949247.413463:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949247.413465:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949247.413473:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f7efedb6. 00000100:00100000:1.0:1784949247.413484:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000008db39772 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289684352:12345-0@lo:400: 00000100:00000001:0.0:1784949247.413490:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:1.0:1784949247.413497:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684352 00000800:00000001:0.0:1784949247.413506:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949247.413507:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:0.0:1784949247.413516:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949247.413519:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949247.413526:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949247.413533:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949247.413537:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949247.413546:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949247.413550:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949247.413551:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:1.0:1784949247.413557:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00100000:2.0:1784949247.413558:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684480 02000000:00000001:2.0:1784949247.413566:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:1.0:1784949247.413567:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:2.0:1784949247.413569:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949247.413576:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949247.413577:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949247.413580:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949247.413586:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:2.0:1784949247.413588:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684480 00000020:00000001:2.0:1784949247.413592:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949247.413594:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949247.413596:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949247.413599:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949247.413603:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949247.413607:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949247.413614:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784949247.413615:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:1.0:1784949247.413624:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000010:2.0:1784949247.413626:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006a06b5eb. 00000100:00000001:2.0:1784949247.413633:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949247.413636:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006a06b5eb. 02000000:00000010:1.0:1784949247.413638:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000579609a4. 00000020:00000010:2.0:1784949247.413646:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000032fe57fd. 02000000:00000001:1.0:1784949247.413647:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949247.413656:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bfe285f8. 00000100:00000001:1.0:1784949247.413658:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949247.413666:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000020:00000001:1.0:1784949247.413668:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949247.413677:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684352 00000100:00000040:2.0:1784949247.413678:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00010000:00000001:1.0:1784949247.413689:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949247.413690:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949247.413697:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949247.413701:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:1.0:1784949247.413703:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000008db39772 x1871651289684352/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:153/0 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949247.413709:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949247.413715:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949247.413719:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949247.413736:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949247.413741:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.413742:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:1.0:1784949247.413754:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=119 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949247.413761:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949247.413766:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949247.413767:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:1.0:1784949247.413778:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00100000:2.0:1784949247.413783:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 786 00000100:00000001:1.0:1784949247.413792:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000040:2.0:1784949247.413793:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949247.413799:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469568128 : -115544239983488 : ffff96e9c29eb480) 02000000:00000001:1.0:1784949247.413805:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:2.0:1784949247.413811:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000de534d9b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:153/0 lens 224/0 e 0 to 0 dl 1784949258 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949247.413814:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949247.413827:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949247.413833:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949247.413838:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006a06b5eb. 00000100:00000040:1.0:1784949247.413843:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949247.413845:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949247.413852:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006a06b5eb. 00000400:00000010:1.0:1784949247.413856:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00100000:2.0:1784949247.413862:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000de534d9b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289684480:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949247.413868:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289684352, offset 224 00000100:00000200:2.0:1784949247.413870:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684480 00000020:00000001:2.0:1784949247.413877:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949247.413884:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949247.413887:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949247.413891:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:2.0:1784949247.413893:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949247.413898:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949247.413960:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949247.413964:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949247.413968:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949247.413970:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949247.413973:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949247.413981:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:1.0:1784949247.413982:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949247.413987:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949247.413992:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949247.413999:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d10efcec. 00000400:00000200:1.0:1784949247.414007:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643d980 02000000:00000001:2.0:1784949247.414007:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949247.414015:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949247.414020:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949247.414023:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684480 00000400:00000200:1.0:1784949247.414029:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x93a5 [1] + 224 00010000:00000001:2.0:1784949247.414031:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949247.414042:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000de534d9b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:153/0 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949247.414050:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:2.0:1784949247.414056:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949247.414061:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949247.414068:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00001000:2.0:1784949247.414070:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=114 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949247.414083:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949247.414086:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1784949247.414088:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949247.414095:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949247.414098:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949247.414102:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949247.414103:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949247.414108:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949247.414118:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949247.414124:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000100:00000200:2.0:1784949247.414134:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289684480, offset 224 00000400:00000200:2.0:1784949247.414148:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949247.414152:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949247.414166:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949247.414185:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3797:3797:241:4294967295] 192.168.202.137@tcp LPNI seq info [2432:2432:8:4294967295] 00000400:00000200:2.0:1784949247.414203:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949247.414208:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949247.414213:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 00000100:00000001:1.0:1784949247.414215:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000200:2.0:1784949247.414223:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949247.414233:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000200:1.0:1784949247.414235:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000800:00000200:2.0:1784949247.414239:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:1.0:1784949247.414246:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784949247.414256:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949247.414260:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949247.414265:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949247.414268:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949247.414277:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000040:1.0:1784949247.414283:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784949247.414286:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949247.414292:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949247.414295:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949247.414298:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:1.0:1784949247.414303:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949247.414312:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949247.414312:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000de534d9b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:153/0 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949247.414328:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008db39772 x1871651289684352/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:153/0 lens 224/224 e 0 to 0 dl 1784949258 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949247.414328:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949247.414335:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1784949247.414343:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000de534d9b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289684480:12345-192.168.202.137@tcp:400: Request processed in 500us (882us total) trans 0 rc 0/0 00000400:00000001:3.0:1784949247.414348:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949247.414354:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949247.414362:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000006138467e. 00000400:00000200:3.0:1784949247.414374:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000100:00100000:2.0:1784949247.414378:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 786 00000400:00000010:3.0:1784949247.414382:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000040:2.0:1784949247.414385:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00100000:1.0:1784949247.414390:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000008db39772 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289684352:12345-0@lo:400: Request processed in 943us (1677us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949247.414393:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949247.414394:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949247.414403:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:3.0:1784949247.414408:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949247.414411:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bfe285f8. 00000100:00100000:1.0:1784949247.414418:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 810 00000020:00000010:2.0:1784949247.414425:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000040:1.0:1784949247.414431:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000010:2.0:1784949247.414438:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000032fe57fd. 00000100:00000001:1.0:1784949247.414442:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949247.414451:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000040:2.0:1784949247.414453:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000800:00000200:0.0:1784949247.414459:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000200:3.0:1784949247.414460:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949247.414461:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000052e9381a. 00000100:00000001:2.0:1784949247.414463:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784949247.414472:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000020:00000010:1.0:1784949247.414479:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000400:00000200:0.0:1784949247.414489:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1784949247.414492:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000b241612. 00000100:00000001:3.0:1784949247.414493:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949247.414501:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784949247.414506:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:0.0:1784949247.414507:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949247.414509:0:337120:0:(client.c:1580:after_reply()) Process entered 00000800:00000001:2.0:1784949247.414515:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949247.414517:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:1.0:1784949247.414521:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949247.414524:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:0.0:1784949247.414525:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 02000000:00000010:1.0:1784949247.414527:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000579609a4. 00000100:00000001:3.0:1784949247.414530:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949247.414532:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949247.414533:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 02000000:00000001:1.0:1784949247.414537:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:3.0:1784949247.414542:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949247.414545:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949247.414547:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949247.414550:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00001000:3.0:1784949247.414551:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=119 v=5 (5 5 5 5) 00000800:00000001:2.0:1784949247.414554:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949247.414557:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784949247.414559:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00001000:3.0:1784949247.414570:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=119 v=5 (1 1 1 1) 00000400:00000010:1.0:1784949247.414573:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000eed58f44. 00000400:00000200:2.0:1784949247.414582:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949247.414583:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:1.0:1784949247.414588:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b92e9146. 00000100:00000001:3.0:1784949247.414590:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949247.414600:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949247.414604:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949247.414613:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643da00 00000100:00000001:3.0:1784949247.414614:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949247.414627:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949247.414637:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x93ad [1] + 224 00000800:00000001:2.0:1784949247.414655:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949247.414658:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949247.414662:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949247.414665:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949247.414671:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949247.414672:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949247.414682:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949247.414697:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:3.0:1784949247.414700:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289684352:0@lo:400: 00000100:00000001:2.0:1784949247.414707:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949247.414709:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949247.414715:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:2.0:1784949247.414721:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949247.414722:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006e540bde x1871651289684352/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949247.414772:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949247.414777:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949247.414781:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000098dd29a0. 00000100:00000040:2.0:1784949247.414784:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949247.414790:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949247.414795:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949247.414800:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949247.414806:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949247.414809:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 02000000:00000001:3.0:1784949247.414814:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949247.414816:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949247.414818:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006e540bde. 00000100:00000001:2.0:1784949247.414819:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949247.414824:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949247.414828:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949247.414830:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949247.414831:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949247.414840:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949247.414843:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949247.414848:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949247.414852:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949247.414856:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949247.414866:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949247.414872:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949247.414880:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949247.414882:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949247.414885:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000948ad730. 00000400:00000200:0.0:1784949247.414892:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000400:00000010:0.0:1784949247.414894:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000100:00000001:0.0:1784949247.414898:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949247.414959:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949247.414973:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949247.414984:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949247.414987:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949247.414989:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949247.414992:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949247.414994:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949247.414997:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949247.414999:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949247.415006:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=114 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949247.415010:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=114 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949247.415013:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949247.415014:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949247.415017:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949247.415021:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949247.415030:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949247.415034:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949247.415036:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949247.415038:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949247.415040:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949247.415044:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949247.415052:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949247.415055:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949247.415057:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949247.415060:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949247.415069:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289684480:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949247.415074:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949247.415077:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949247.415080:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289684480/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949247.415096:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949247.415098:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949247.415101:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949247.415106:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949247.415108:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949247.415109:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949247.415112:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949247.415114:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 02000000:00000001:0.0:1784949247.415118:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949247.415120:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949247.415121:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:0.0:1784949247.415125:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949247.415127:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949247.415129:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949247.415134:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949247.415136:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949247.415138:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949247.415141:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949247.456025:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306186795, at 4306186796 00000100:00000010:1.0:1784949247.456039:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000040:3.0:1784949247.456051:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306186795, at 4306186796 00000100:00000040:0.0:1784949247.456055:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306186795, at 4306186796 00000100:00000040:2.0:1784949247.456058:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306186795, at 4306186796 00000100:00000010:3.0:1784949247.456066:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:0.0:1784949247.456074:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000010:2.0:1784949247.456079:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00010000:00000001:2.0:1784949248.244019:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949248.244029:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949248.244037:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949248.244039:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949248.244180:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949248.244187:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949248.244192:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949248.436048:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949248.436068:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949248.436072:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949248.436081:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949248.436085:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949248.436091:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949248.436097:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949249.268066:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949249.268079:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949249.268091:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949249.268095:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949249.268182:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949249.268188:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949249.268193:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949249.460008:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949249.460021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949249.460023:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949249.460028:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949249.460031:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949249.460034:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949249.460037:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949250.292004:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949250.292012:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949250.292019:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949250.292021:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949250.292094:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949250.292101:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949250.292106:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949250.484082:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949250.484099:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949250.484104:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949250.484110:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949250.484114:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949250.484119:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949250.484124:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949251.316036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949251.316048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949251.316059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949251.316065:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949251.316069:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949251.316075:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949251.316080:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949251.316083:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949251.316087:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949251.316092:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949251.316175:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949251.316182:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949251.316187:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949251.508122:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949251.508139:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949251.508142:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949251.508150:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949251.508154:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949251.508160:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949251.508164:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949252.340034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949252.340045:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949252.340057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949252.340060:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949252.340143:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949252.340151:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949252.340155:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949252.532042:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1784949252.532046:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949252.532058:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949252.532064:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:3.0:1784949252.532068:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006e540bde. 00000100:00000001:2.0:1784949252.532071:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784949252.532081:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949252.532083:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949252.532093:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949252.532094:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949252.532101:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949252.532102:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949252.532111:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949252.532112:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949252.532127:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:3.0:1784949252.532137:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949252.532150:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949252.532168:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949252.532171:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949252.532177:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e540bde] to pc [ptlrpcd_00_03+3] req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949252.532212:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949252.532215:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949252.532220:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949252.532224:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949252.532229:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949252.532233:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000020:00000040:3.0:1784949252.532237:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949252.532240:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949252.532242:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949252.532244:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.532249:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784949252.532249:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:0.0:1784949252.532259:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949252.532260:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.532270:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949252.532276:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784949252.532288:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949252.532293:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949252.532298:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949252.532312:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_00_00+0] req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949252.532324:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949252.532335:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.532336:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949252.532347:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949252.532358:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949252.532364:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949252.532371:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949252.532372:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949252.532380:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289684608:0@lo:400: 00000100:00000040:3.0:1784949252.532383:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11529) 00000100:00000001:1.0:1784949252.532394:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949252.532396:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1784949252.532405:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 02000000:00000001:0.0:1784949252.532407:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949252.532411:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949252.532414:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949252.532417:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949252.532422:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949252.532423:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949252.532426:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949252.532431:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784949252.532436:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 00000100:00000001:1.0:1784949252.532437:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949252.532442:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949252.532445:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949252.532448:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000010:0.0:1784949252.532455:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000948ad730. 00000100:00000040:1.0:1784949252.532462:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949252.532470:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000100:00000001:1.0:1784949252.532485:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:0.0:1784949252.532486:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684608, portal 25 00000100:00000001:1.0:1784949252.532489:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.532495:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1784949252.532498:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949252.532501:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554957056 : -115544154594560 : ffff96e9c7b5a300) 02000000:00000001:1.0:1784949252.532504:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949252.532511:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949252.532515:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949252.532523:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289684736:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949252.532542:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949252.532548:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949252.532550:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949252.532552:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949252.532558:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 02000000:00000001:1.0:1784949252.532566:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.532569:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784949252.532575:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000040:0.0:1784949252.532577:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949252.532589:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00000400:00000010:1.0:1784949252.532591:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000200:0.0:1784949252.532595:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289684608, offset 0 00000100:00000200:1.0:1784949252.532601:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684736, portal 10 00000100:00000001:1.0:1784949252.532609:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1784949252.532610:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949252.532613:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554960640 : -115544154590976 : ffff96e9c7b5b100) 00000100:00000040:1.0:1784949252.532625:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949252.532640:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949252.532654:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949252.532660:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949252.532666:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000400:00000200:0.0:1784949252.532667:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643da80 00000100:00000200:1.0:1784949252.532674:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289684736, offset 0 00000400:00000200:0.0:1784949252.532682:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8a25 [2] + 0 00000400:00000010:0.0:1784949252.532689:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c4db7024. 00000400:00000200:0.0:1784949252.532697:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006a025321 00000400:00000200:1.0:1784949252.532706:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949252.532708:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949252.532716:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949252.532725:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949252.532732:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949252.532739:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949252.532743:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949252.532748:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949252.532752:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3798:3798:241:4294967295] 192.168.202.137@tcp LPNI seq info [2433:2433:8:4294967295] 00000400:00000200:1.0:1784949252.532767:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949252.532775:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949252.532776:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006a025321 00000800:00000010:1.0:1784949252.532782:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008d5370ec. 00000400:00000010:0.0:1784949252.532784:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000006a025321. 00000100:00000001:0.0:1784949252.532797:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:1.0:1784949252.532804:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784949252.532805:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949252.532815:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000875035e8 x1871651289684608 msgsize 224 00000100:00100000:0.0:1784949252.532833:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000200:1.0:1784949252.532834:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:0.0:1784949252.532838:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:1.0:1784949252.532846:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008d5370ec type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949252.532897:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949252.532901:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949252.532950:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdaa2431 00000400:00000010:0.0:1784949252.532958:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdaa2431. 00000100:00000040:1.0:1784949252.532961:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2232 usecs (3 jiffies) 00000100:00000001:0.0:1784949252.532964:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949252.532966:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949252.532969:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306191876 00000100:00100000:3.0:1784949252.532975:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684608 00000100:00000200:0.0:1784949252.532977:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949252.532982:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 02000000:00000001:3.0:1784949252.532984:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949252.532993:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949252.532999:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949252.533001:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 02000000:00000001:3.0:1784949252.533015:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949252.533019:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306191876 00000100:00000001:0.0:1784949252.533024:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1784949252.533025:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684608 00000100:00000010:1.0:1784949252.533029:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000020:00000001:3.0:1784949252.533032:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:0.0:1784949252.533034:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949252.533040:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949252.533041:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000020:00000001:3.0:1784949252.533052:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949252.533053:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306191876 00000020:00000040:3.0:1784949252.533061:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949252.533062:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949252.533063:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000020:00000001:3.0:1784949252.533076:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949252.533077:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949252.533079:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000001:3.0:1784949252.533087:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949252.533091:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949252.533093:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306191876 00000100:00000001:3.0:1784949252.533096:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949252.533101:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306191915 00000800:00000200:2.0:1784949252.533107:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1784949252.533108:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b3f939e4. 00000100:00000010:1.0:1784949252.533109:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:0.0:1784949252.533114:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306191876 00000100:00000001:3.0:1784949252.533123:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1784949252.533124:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008d5370ec. 00000100:00000001:1.0:1784949252.533125:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949252.533128:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306191915 00000020:00000010:3.0:1784949252.533140:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b3f939e4. 00000100:00000001:1.0:1784949252.533141:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949252.533143:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306191876 00000400:00000200:2.0:1784949252.533145:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949252.533156:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949252.533157:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000072ce5ef0. 00000100:00000001:0.0:1784949252.533160:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306191915 00000100:00000001:1.0:1784949252.533164:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949252.533171:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008cc004d6. 00000400:00000200:2.0:1784949252.533174:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949252.533177:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.533178:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306191876 00000020:00000010:3.0:1784949252.533185:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000200:2.0:1784949252.533190:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000100:00000001:0.0:1784949252.533194:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306191915 00000400:00000010:2.0:1784949252.533197:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000100:00000040:3.0:1784949252.533206:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949252.533208:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949252.533209:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306191876 00000100:00000001:3.0:1784949252.533218:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949252.533223:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784949252.533225:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949252.533227:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000001:1.0:1784949252.533229:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949252.533230:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949252.533234:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.533243:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949252.533250:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949252.533252:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.533254:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949252.533261:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949252.533270:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949252.533273:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949252.533274:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949252.533278:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949252.533282:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:2.0:1784949252.533290:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949252.533294:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949252.533299:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949252.533302:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949252.533308:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949252.533311:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949252.533319:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949252.533321:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949252.533328:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643db00 00000100:00100000:3.0:1784949252.533331:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 811 00000100:00000001:2.0:1784949252.533334:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949252.533336:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949252.533347:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:1.0:1784949252.533352:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 177328 00000100:00000001:3.0:1784949252.533355:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466827856 : -115544242723760 : ffff96e9c274e450) 00000800:00000001:1.0:1784949252.533367:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949252.533370:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000875035e8 x1871651289684608/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:158/0 lens 224/0 e 0 to 0 dl 1784949263 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949252.533379:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949252.533384:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784949252.533386:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949252.533393:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b3f939e4. 00000400:00000200:1.0:1784949252.533398:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949252.533402:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949252.533407:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b3f939e4. 00000100:00000001:1.0:1784949252.533410:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949252.533414:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00100000:3.0:1784949252.533421:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000875035e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289684608:12345-0@lo:400: 00000100:00000010:1.0:1784949252.533423:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008c05042f. 00000100:00000200:3.0:1784949252.533434:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684608 00000100:00000040:1.0:1784949252.533435:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000008c05042f x1871651289684736 msgsize 224 00000020:00000001:3.0:1784949252.533442:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:1.0:1784949252.533451:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:3.0:1784949252.533452:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949252.533459:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949252.533463:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949252.533466:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949252.533471:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949252.533473:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949252.533476:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:3.0:1784949252.533477:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949252.533485:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949252.533488:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949252.533490:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:1.0:1784949252.533491:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949252.533495:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000001:1.0:1784949252.533500:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949252.533501:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949252.533512:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d78419ed. 02000000:00000001:3.0:1784949252.533518:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949252.533521:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949252.533525:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949252.533530:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000040:3.0:1784949252.533530:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684608 00000100:00100000:2.0:1784949252.533536:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684736 00010000:00000001:3.0:1784949252.533539:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:2.0:1784949252.533544:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949252.533548:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949252.533550:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949252.533555:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949252.533558:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684736 00000020:00000001:2.0:1784949252.533562:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00010000:00000200:3.0:1784949252.533567:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000875035e8 x1871651289684608/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:158/0 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784949252.533568:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949252.533575:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949252.533580:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949252.533588:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949252.533596:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:3.0:1784949252.533601:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949252.533604:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:3.0:1784949252.533606:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949252.533612:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=124 v=5 (1 1 1 1) 00000020:00000010:2.0:1784949252.533613:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:2.0:1784949252.533623:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949252.533626:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1784949252.533630:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000040:3.0:1784949252.533634:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949252.533640:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000010:2.0:1784949252.533642:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000032fe57fd. 02000000:00000001:3.0:1784949252.533644:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949252.533651:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949252.533652:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bfe285f8. 00000100:00000001:3.0:1784949252.533655:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:2.0:1784949252.533660:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000040:3.0:1784949252.533662:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949252.533672:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000100:00000040:2.0:1784949252.533674:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000200:3.0:1784949252.533678:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289684608, offset 224 00000100:00000001:2.0:1784949252.533680:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949252.533687:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:3.0:1784949252.533691:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949252.533692:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949252.533704:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:3.0:1784949252.533712:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949252.533713:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949252.533718:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949252.533730:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643da80 00000400:00000200:3.0:1784949252.533738:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x93c1 [1] + 224 00000100:00000001:2.0:1784949252.533740:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949252.533746:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949252.533753:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949252.533758:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949252.533760:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949252.533763:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:3.0:1784949252.533770:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949252.533778:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 787 00000100:00000040:2.0:1784949252.533789:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949252.533798:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637808256 : -115544071743360 : ffff96e9cca5d880) 00000100:00000040:3.0:1784949252.533804:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949252.533810:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000008c05042f x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:158/0 lens 224/0 e 0 to 0 dl 1784949263 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949252.533887:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949252.533888:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784949252.533893:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:2.0:1784949252.533900:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949252.533901:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000100:00000001:0.0:1784949252.533935:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949252.533941:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:3.0:1784949252.533947:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000100:00000001:3.0:1784949252.533953:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000010:2.0:1784949252.533960:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000001:3.0:1784949252.533960:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:0.0:1784949252.533966:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949252.533972:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949252.533973:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949252.533979:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000008c05042f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289684736:12345-192.168.202.137@tcp:400: 00000400:00000010:0.0:1784949252.533981:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000948ad730. 00000100:00000040:3.0:1784949252.533986:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000200:0.0:1784949252.533995:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000100:00000200:2.0:1784949252.533997:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684736 00010000:00000001:3.0:1784949252.533998:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:0.0:1784949252.534003:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000020:00000001:3.0:1784949252.534006:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949252.534007:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949252.534011:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784949252.534015:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:2.0:1784949252.534016:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:0.0:1784949252.534023:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949252.534024:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949252.534029:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000875035e8 x1871651289684608/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:158/0 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949252.534036:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949252.534045:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949252.534059:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949252.534068:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949252.534080:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949252.534090:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:0.0:1784949252.534095:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949252.534099:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000875035e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289684608:12345-0@lo:400: Request processed in 707us (1274us total) trans 0 rc 0/0 00000020:00000001:2.0:1784949252.534100:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949252.534113:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949252.534122:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949252.534131:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949252.534134:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1784949252.534136:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 811 00000100:00000001:0.0:1784949252.534142:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784949252.534144:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006f496819. 00000100:00000040:3.0:1784949252.534145:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949252.534148:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949252.534157:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949252.534158:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:0.0:1784949252.534160:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949252.534168:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949252.534169:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949252.534170:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.534177:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949252.534179:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008cc004d6. 00000020:00000001:2.0:1784949252.534180:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949252.534184:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949252.534192:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684736 00000100:00001000:0.0:1784949252.534195:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=124 v=5 (5 5 5 5) 00000020:00000010:3.0:1784949252.534195:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00010000:00000001:2.0:1784949252.534198:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00001000:0.0:1784949252.534209:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=124 v=5 (1 1 1 1) 00010000:00000200:2.0:1784949252.534210:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000008c05042f x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:158/0 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949252.534212:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000072ce5ef0. 00000100:00000001:0.0:1784949252.534225:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000040:3.0:1784949252.534230:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784949252.534232:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949252.534240:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949252.534246:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949252.534247:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949252.534248:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:0.0:1784949252.534253:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949252.534259:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=119 v=5 (1 1 1 1) 02000000:00000010:3.0:1784949252.534260:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d78419ed. 00000100:00000001:0.0:1784949252.534261:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949252.534276:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949252.534277:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949252.534283:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949252.534287:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949252.534289:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949252.534300:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000200:3.0:1784949252.534306:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:2.0:1784949252.534308:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949252.534314:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949252.534323:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:3.0:1784949252.534324:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000af4b2538. 00000100:00000040:2.0:1784949252.534337:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949252.534340:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784949252.534344:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e9d273c9. 00000400:00000010:2.0:1784949252.534345:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000100:00000001:0.0:1784949252.534350:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949252.534359:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949252.534362:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289684736, offset 224 00000100:00000040:0.0:1784949252.534371:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949252.534377:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784949252.534396:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006e540bde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289684608:0@lo:400: 00000400:00000200:2.0:1784949252.534398:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949252.534417:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949252.534418:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3799:3799:241:4294967295] 192.168.202.137@tcp LPNI seq info [2434:2434:8:4294967295] 00000100:00000001:0.0:1784949252.534425:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949252.534433:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006e540bde x1871651289684608/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949252.534454:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949252.534466:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949252.534471:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:2.0:1784949252.534472:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000010:0.0:1784949252.534475:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 00000800:00000010:2.0:1784949252.534480:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 02000000:00000001:0.0:1784949252.534485:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000200:2.0:1784949252.534493:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:0.0:1784949252.534494:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949252.534499:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949252.534505:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000800:00000200:2.0:1784949252.534508:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000010:0.0:1784949252.534510:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 00000800:00000200:2.0:1784949252.534517:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 1, nob 320 niov 1 nkiov 1 02000000:00000001:0.0:1784949252.534521:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949252.534526:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949252.534528:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006e540bde. 00000100:00000001:0.0:1784949252.534534:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949252.534535:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949252.534538:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949252.534541:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949252.534548:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949252.534551:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949252.534553:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949252.534556:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949252.534562:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008c05042f x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:158/0 lens 224/224 e 0 to 0 dl 1784949263 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949252.534563:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949252.534569:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949252.534580:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949252.534588:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949252.534594:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000008c05042f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289684736:12345-192.168.202.137@tcp:400: Request processed in 730us (1152us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949252.534609:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 787 00000100:00000040:2.0:1784949252.534614:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949252.534617:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949252.534620:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949252.534624:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bfe285f8. 00000020:00000010:2.0:1784949252.534629:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949252.534633:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000032fe57fd. 00000020:00000040:2.0:1784949252.534638:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949252.534641:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949252.534694:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949252.534697:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949252.534701:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000400:00000200:3.0:1784949252.534709:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949252.534711:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949252.534718:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949252.534720:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784949252.534724:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949252.534728:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000400:00000010:3.0:1784949252.534737:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000400:00000200:2.0:1784949252.534741:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949252.534744:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949252.534750:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949252.534757:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643db00 00000400:00000200:2.0:1784949252.534766:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x93c9 [1] + 224 00000800:00000001:2.0:1784949252.534773:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949252.534786:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949252.534789:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949252.534793:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949252.534799:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949252.534803:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949252.534814:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949252.534844:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949252.534851:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949252.534854:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949252.534935:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949252.534940:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949252.534946:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949252.534949:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949252.534951:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000400:00000200:1.0:1784949252.534957:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:1.0:1784949252.534960:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:1.0:1784949252.534964:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949252.534970:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949252.534982:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949252.534991:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949252.534993:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949252.534995:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949252.534998:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949252.534999:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949252.535001:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949252.535004:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949252.535011:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=119 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949252.535015:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=119 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949252.535018:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949252.535019:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949252.535022:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949252.535026:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949252.535035:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949252.535038:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949252.535040:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949252.535042:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949252.535045:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949252.535048:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949252.535057:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949252.535059:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949252.535061:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949252.535064:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949252.535073:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289684736:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949252.535077:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949252.535080:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949252.535082:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289684736/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949252.535090:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949252.535092:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949252.535094:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:1.0:1784949252.535097:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949252.535099:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949252.535101:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949252.535103:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949252.535105:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:1.0:1784949252.535109:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949252.535110:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949252.535112:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000100:00000001:1.0:1784949252.535115:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949252.535117:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949252.535119:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949252.535124:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949252.535126:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949252.535128:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949252.535130:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949252.576014:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306191915, at 4306191916 00000100:00000010:2.0:1784949252.576030:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000100:00000040:3.0:1784949252.576044:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306191915, at 4306191916 00000100:00000040:0.0:1784949252.576049:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306191915, at 4306191916 00000100:00000040:1.0:1784949252.576054:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306191915, at 4306191916 00000100:00000010:3.0:1784949252.576064:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:0.0:1784949252.576069:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:1.0:1784949252.576073:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949253.364026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949253.364038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949253.364050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949253.364053:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949253.364147:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949253.364154:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949253.364158:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949253.556018:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949253.556036:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949253.556040:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949253.556048:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949253.556052:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949253.556057:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949253.556062:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949254.388048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949254.388060:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949254.388072:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949254.388074:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949254.388204:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949254.388211:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949254.388216:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949254.580100:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949254.580117:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949254.580120:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949254.580125:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949254.580128:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949254.580133:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949254.580137:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949255.412040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949255.412049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949255.412057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949255.412059:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949255.412153:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949255.412159:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949255.412164:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949255.604015:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949255.604033:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949255.604037:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949255.604044:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949255.604048:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949255.604053:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949255.604057:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949256.436045:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949256.436055:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949256.436062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949256.436064:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949256.436173:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949256.436182:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949256.436188:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949256.628030:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949256.628047:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949256.628049:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949256.628056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949256.628059:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949256.628064:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949256.628067:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949257.459980:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949257.459986:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949257.459991:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949257.459992:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949257.460044:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949257.460049:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949257.460051:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1784949257.652050:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949257.652063:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949257.652073:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000010:3.0:1784949257.652076:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000020:00000040:3.0:1784949257.652087:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949257.652090:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949257.652096:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949257.652096:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949257.652107:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949257.652108:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949257.652115:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949257.652118:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949257.652120:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949257.652127:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949257.652134:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:3.0:1784949257.652146:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949257.652159:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949257.652175:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949257.652179:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949257.652187:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006d868603] to pc [ptlrpcd_00_01+1] req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949257.652220:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949257.652224:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949257.652229:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949257.652234:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949257.652238:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949257.652241:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046cb422a. 00000020:00000040:3.0:1784949257.652245:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949257.652250:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949257.652250:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949257.652253:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949257.652258:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.652263:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1784949257.652265:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000001:3.0:1784949257.652272:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.652273:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949257.652282:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949257.652289:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949257.652291:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949257.652302:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.652316:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949257.652322:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.652327:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949257.652333:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949257.652334:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046cb422a] to pc [ptlrpcd_00_02+2] req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949257.652344:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949257.652352:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949257.652357:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949257.652362:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289684864:0@lo:400: 00000100:00000001:1.0:1784949257.652368:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949257.652372:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949257.652378:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784949257.652382:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11534) 02000000:00000001:1.0:1784949257.652384:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949257.652386:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949257.652394:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949257.652396:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949257.652402:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1784949257.652405:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 00000100:00000040:2.0:1784949257.652414:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949257.652415:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949257.652427:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000001:0.0:1784949257.652430:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949257.652439:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949257.652442:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949257.652444:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949257.652445:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 02000000:00000001:2.0:1784949257.652458:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000200:1.0:1784949257.652462:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684864, portal 25 02000000:00000001:2.0:1784949257.652464:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949257.652473:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1784949257.652474:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.652480:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554951680 : -115544154599936 : ffff96e9c7b58e00) 00000100:00100000:2.0:1784949257.652484:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289684992:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949257.652492:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949257.652494:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949257.652498:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949257.652505:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949257.652513:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949257.652520:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:2.0:1784949257.652523:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000032fe57fd. 00000100:00000040:1.0:1784949257.652529:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784949257.652531:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949257.652536:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000400:00000010:2.0:1784949257.652541:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000075224c9. 00000100:00000200:1.0:1784949257.652543:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289684864, offset 0 00000400:00000010:2.0:1784949257.652557:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000200:1.0:1784949257.652559:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000200:2.0:1784949257.652578:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289684992, portal 10 00000400:00000200:1.0:1784949257.652589:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949257.652590:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949257.652596:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554953472 : -115544154598144 : ffff96e9c7b59500) 00000100:00000040:2.0:1784949257.652608:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949257.652619:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643db80 00000100:00000001:2.0:1784949257.652634:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784949257.652638:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8a4d [2] + 0 00000100:00000040:2.0:1784949257.652647:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949257.652651:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000026bb7060. 00000400:00000010:2.0:1784949257.652655:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000200:1.0:1784949257.652661:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000016d64ec5 00000100:00000200:2.0:1784949257.652669:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289684992, offset 0 00000400:00000200:1.0:1784949257.652674:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949257.652683:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949257.652690:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784949257.652704:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949257.652712:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949257.652718:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949257.652722:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949257.652726:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949257.652730:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3800:3800:241:4294967295] 192.168.202.137@tcp LPNI seq info [2435:2435:8:4294967295] 00000400:00000200:2.0:1784949257.652744:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949257.652752:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949257.652754:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000016d64ec5 00000800:00000010:2.0:1784949257.652759:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 00000400:00000010:1.0:1784949257.652761:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000016d64ec5. 00000100:00000001:1.0:1784949257.652772:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:2.0:1784949257.652776:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:1.0:1784949257.652778:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:1.0:1784949257.652790:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000019a655f1 x1871651289684864 msgsize 224 00000800:00000200:2.0:1784949257.652802:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:1.0:1784949257.652804:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000200:2.0:1784949257.652837:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:1.0:1784949257.652845:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949257.652863:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949257.652872:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2233 usecs (3 jiffies) 00000100:00000001:1.0:1784949257.652874:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949257.652878:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306196995 00000100:00000010:2.0:1784949257.652886:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000400:00000200:1.0:1784949257.652889:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000010:1.0:1784949257.652894:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000100:00000001:2.0:1784949257.652896:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949257.652900:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949257.652949:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949257.652951:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306196996 00000100:00000010:2.0:1784949257.652961:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001cae17e9. 00000100:00000001:3.0:1784949257.652964:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949257.652973:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00100000:3.0:1784949257.652977:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684864 00000100:00000001:1.0:1784949257.652980:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949257.652983:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306196996 02000000:00000001:3.0:1784949257.652987:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000010:2.0:1784949257.652991:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e19b789f. 00000100:00000040:1.0:1784949257.652993:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949257.652996:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949257.652999:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949257.653007:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949257.653010:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306196996 00000100:00000010:2.0:1784949257.653024:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000ef8f137f. 02000000:00000001:3.0:1784949257.653024:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.653026:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949257.653036:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949257.653038:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1784949257.653040:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684864 00000100:00000001:2.0:1784949257.653047:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949257.653049:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784949257.653052:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949257.653056:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949257.653058:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949257.653061:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306197035 00000100:00000001:2.0:1784949257.653063:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949257.653067:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949257.653074:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784949257.653078:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949257.653079:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306196995 00000100:00000001:2.0:1784949257.653085:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949257.653094:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000001:3.0:1784949257.653094:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949257.653095:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306197035 00000100:00000001:2.0:1784949257.653109:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949257.653110:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949257.653113:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306196996 00000100:00000001:2.0:1784949257.653115:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:0.0:1784949257.653117:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949257.653124:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.653127:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000010:0.0:1784949257.653129:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000100:00000001:1.0:1784949257.653129:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306197035 00000100:00000001:2.0:1784949257.653131:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949257.653141:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dd6c0379. 00000400:00000200:0.0:1784949257.653147:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949257.653150:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306196996 00000100:00000001:3.0:1784949257.653151:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949257.653163:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:3.0:1784949257.653166:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dd6c0379. 00000100:00000001:1.0:1784949257.653169:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306197035 00000400:00000200:0.0:1784949257.653175:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:0.0:1784949257.653182:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000800:00000001:2.0:1784949257.653185:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784949257.653186:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bf393d23. 00000100:00000001:1.0:1784949257.653187:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306196996 00000100:00000001:0.0:1784949257.653189:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:3.0:1784949257.653202:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008cc004d6. 00000100:00000200:0.0:1784949257.653203:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949257.653204:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949257.653209:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1784949257.653217:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949257.653218:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949257.653223:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949257.653235:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949257.653240:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:2.0:1784949257.653240:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949257.653243:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949257.653243:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000040:0.0:1784949257.653253:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949257.653257:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949257.653262:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949257.653266:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949257.653270:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:2.0:1784949257.653276:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949257.653277:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.653281:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949257.653287:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949257.653294:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.653299:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.653304:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949257.653306:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949257.653315:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643dc00 00000100:00000001:0.0:1784949257.653318:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949257.653329:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949257.653335:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 177552 00000800:00000001:2.0:1784949257.653348:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949257.653361:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:2.0:1784949257.653362:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949257.653370:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949257.653371:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949257.653372:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949257.653378:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949257.653388:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:3.0:1784949257.653392:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 812 00000100:00000001:2.0:1784949257.653398:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784949257.653403:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949257.653404:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949257.653404:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784949257.653417:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.653418:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529466821712 : -115544242729904 : ffff96e9c274cc50) 00000100:00000010:2.0:1784949257.653419:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004ef1e045. 00000100:00000040:2.0:1784949257.653430:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000004ef1e045 x1871651289684992 msgsize 224 00000100:00000040:3.0:1784949257.653439:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000019a655f1 x1871651289684864/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:163/0 lens 224/0 e 0 to 0 dl 1784949268 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949257.653448:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784949257.653470:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949257.653478:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000010:3.0:1784949257.653478:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dd6c0379. 00000100:00000001:3.0:1784949257.653487:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949257.653494:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:3.0:1784949257.653495:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dd6c0379. 00000800:00000001:2.0:1784949257.653504:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949257.653515:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000019a655f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289684864:12345-0@lo:400: 00000100:00000200:3.0:1784949257.653525:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684864 00000020:00000001:3.0:1784949257.653533:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949257.653540:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:0.0:1784949257.653542:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:3.0:1784949257.653545:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949257.653548:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289684992 00000020:00000001:3.0:1784949257.653555:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:0.0:1784949257.653556:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949257.653564:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:3.0:1784949257.653564:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784949257.653569:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949257.653576:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 02000000:00000001:0.0:1784949257.653577:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949257.653583:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949257.653585:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289684992 00000020:00000001:0.0:1784949257.653594:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:3.0:1784949257.653595:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000040:0.0:1784949257.653597:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949257.653603:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949257.653604:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949257.653612:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949257.653613:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949257.653616:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949257.653624:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949257.653624:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949257.653633:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949257.653634:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000010:0.0:1784949257.653638:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 02000000:00000001:3.0:1784949257.653642:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949257.653645:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949257.653651:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 02000000:00000010:3.0:1784949257.653653:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c38711c7. 00000020:00000010:0.0:1784949257.653659:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 02000000:00000001:3.0:1784949257.653662:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.653671:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949257.653672:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002eb61d28. 00000020:00000001:3.0:1784949257.653678:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949257.653682:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002ffa19fa. 00010000:00000040:3.0:1784949257.653687:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684864 00000100:00000040:0.0:1784949257.653694:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00010000:00000001:3.0:1784949257.653697:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949257.653700:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949257.653705:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949257.653709:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:3.0:1784949257.653711:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000019a655f1 x1871651289684864/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:163/0 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949257.653718:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949257.653724:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949257.653729:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949257.653743:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949257.653748:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949257.653755:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784949257.653757:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=129 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949257.653772:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949257.653775:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949257.653779:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949257.653781:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949257.653790:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00100000:0.0:1784949257.653796:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 788 02000000:00000001:3.0:1784949257.653806:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000040:0.0:1784949257.653807:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949257.653837:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529562019456 : -115544147532160 : ffff96e9c8216680) 02000000:00000001:3.0:1784949257.653844:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949257.653847:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000004ef1e045 x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:163/0 lens 224/0 e 0 to 0 dl 1784949268 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949257.653858:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949257.653869:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:3.0:1784949257.653870:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:0.0:1784949257.653878:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000400:00000010:3.0:1784949257.653883:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000100:00000001:0.0:1784949257.653885:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949257.653891:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000100:00000200:3.0:1784949257.653899:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289684864, offset 224 00000100:00100000:0.0:1784949257.653941:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000004ef1e045 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289684992:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949257.653950:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289684992 00000020:00000001:0.0:1784949257.653953:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949257.653958:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949257.653964:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949257.653966:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949257.653971:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000200:3.0:1784949257.653973:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:0.0:1784949257.653978:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949257.653984:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949257.653988:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949257.653993:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784949257.653996:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:0.0:1784949257.653998:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949257.654007:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949257.654013:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949257.654017:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:3.0:1784949257.654023:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643db80 02000000:00000010:0.0:1784949257.654025:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002ef2c37b. 02000000:00000001:0.0:1784949257.654033:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949257.654037:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949257.654041:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x93dd [1] + 224 00000020:00000001:0.0:1784949257.654043:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949257.654052:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289684992 00010000:00000001:0.0:1784949257.654058:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949257.654064:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000004ef1e045 x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:163/0 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949257.654064:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949257.654083:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00010000:00000001:0.0:1784949257.654091:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949257.654097:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.654098:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00001000:0.0:1784949257.654101:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=124 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949257.654109:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1784949257.654112:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949257.654117:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949257.654124:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949257.654134:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949257.654139:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949257.654148:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949257.654156:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949257.654161:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949257.654163:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001e880b77. 00000100:00000200:0.0:1784949257.654174:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289684992, offset 224 00000400:00000200:0.0:1784949257.654193:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949257.654211:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949257.654219:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3801:3801:241:4294967295] 192.168.202.137@tcp LPNI seq info [2436:2436:8:4294967295] 00000100:00000001:3.0:1784949257.654221:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949257.654237:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949257.654246:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949257.654251:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000820f02b3. 00000400:00000200:3.0:1784949257.654254:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000800:00000200:0.0:1784949257.654264:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000010:3.0:1784949257.654265:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000100:00000001:1.0:1784949257.654269:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949257.654277:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:0.0:1784949257.654279:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784949257.654280:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000001:1.0:1784949257.654288:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:0.0:1784949257.654289:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000820f02b3 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949257.654289:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:1.0:1784949257.654294:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949257.654301:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000100:00000001:3.0:1784949257.654303:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949257.654312:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000100:00000040:3.0:1784949257.654317:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000010:1.0:1784949257.654319:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:0.0:1784949257.654320:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949257.654330:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000040:0.0:1784949257.654332:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000020:00000001:3.0:1784949257.654338:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949257.654339:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949257.654347:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949257.654350:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784949257.654350:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:0.0:1784949257.654353:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:1.0:1784949257.654365:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949257.654372:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004ef1e045 x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:163/0 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949257.654373:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000019a655f1 x1871651289684864/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:163/0 lens 224/224 e 0 to 0 dl 1784949268 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949257.654426:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949257.654458:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000004ef1e045 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289684992:12345-192.168.202.137@tcp:400: Request processed in 578us (1015us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949257.654462:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000019a655f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289684864:12345-0@lo:400: Request processed in 983us (1660us total) trans 0 rc 0/0 00000800:00000200:2.0:1784949257.654478:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949257.654479:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949257.654488:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949257.654490:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000820f02b3. 00000100:00000001:1.0:1784949257.654498:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00100000:0.0:1784949257.654501:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 788 00000400:00000200:2.0:1784949257.654503:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:3.0:1784949257.654505:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 812 02000000:00000001:1.0:1784949257.654507:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000040:0.0:1784949257.654516:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949257.654517:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:3.0:1784949257.654520:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949257.654524:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949257.654527:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949257.654528:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:1.0:1784949257.654534:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.654535:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949257.654537:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:2.0:1784949257.654544:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001e880b77 00000100:00000001:3.0:1784949257.654546:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00001000:1.0:1784949257.654548:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=129 v=5 (5 5 5 5) 00000020:00000010:0.0:1784949257.654549:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002eb61d28. 00000400:00000010:2.0:1784949257.654551:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001e880b77. 00000020:00000010:3.0:1784949257.654557:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008cc004d6. 00000020:00000010:0.0:1784949257.654566:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002ffa19fa. 00000100:00001000:1.0:1784949257.654567:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=129 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949257.654567:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000010:3.0:1784949257.654571:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:2.0:1784949257.654575:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000010:0.0:1784949257.654582:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000100:00000001:1.0:1784949257.654586:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:3.0:1784949257.654591:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bf393d23. 00000100:00000001:1.0:1784949257.654592:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949257.654593:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000040:0.0:1784949257.654595:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00010000:00000001:1.0:1784949257.654604:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1784949257.654606:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949257.654608:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949257.654610:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949257.654611:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000800:00000001:2.0:1784949257.654620:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949257.654623:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949257.654629:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000800:00000001:2.0:1784949257.654630:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949257.654636:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784949257.654639:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c38711c7. 00000400:00000200:2.0:1784949257.654653:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:3.0:1784949257.654658:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949257.654667:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949257.654681:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643dc00 00000100:00000001:1.0:1784949257.654683:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784949257.654688:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949257.654692:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1784949257.654696:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x93e5 [1] + 224 00000100:00000001:1.0:1784949257.654702:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949257.654705:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a51bb92. 00000800:00000001:2.0:1784949257.654706:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949257.654718:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949257.654720:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949257.654721:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000048b052c7. 00000400:00000200:2.0:1784949257.654731:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949257.654750:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949257.654762:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784949257.654770:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289684864:0@lo:400: 00000100:00000200:2.0:1784949257.654774:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949257.654794:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949257.654801:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949257.654805:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949257.654848:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006d868603 x1871651289684864/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949257.654861:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949257.654867:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:2.0:1784949257.654873:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784949257.654877:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000001:2.0:1784949257.654881:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949257.654886:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:1.0:1784949257.654897:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949257.654902:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949257.654952:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949257.654959:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1784949257.654964:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1784949257.654965:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 00000100:00000001:3.0:1784949257.654971:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1784949257.654978:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000001:3.0:1784949257.654982:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1784949257.654985:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000001:3.0:1784949257.654987:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000010:1.0:1784949257.654991:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006d868603. 00000400:00000010:3.0:1784949257.654993:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000075224c9. 00000400:00000200:3.0:1784949257.655006:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:3.0:1784949257.655009:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000100:00000001:3.0:1784949257.655012:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784949257.655020:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000200:3.0:1784949257.655021:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949257.655029:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949257.655040:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949257.655054:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949257.655057:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949257.655065:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.655077:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949257.655080:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949257.655082:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949257.655090:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.655091:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949257.655096:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949257.655101:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949257.655103:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949257.655106:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949257.655113:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=124 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949257.655117:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=124 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949257.655122:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949257.655124:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949257.655126:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949257.655131:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949257.655141:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.655144:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949257.655146:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949257.655148:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949257.655150:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949257.655154:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949257.655162:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949257.655165:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949257.655167:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949257.655170:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949257.655178:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289684992:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949257.655183:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949257.655186:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949257.655188:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000046cb422a x1871651289684992/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949257.655196:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949257.655198:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949257.655200:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000032fe57fd. 02000000:00000001:3.0:1784949257.655204:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949257.655206:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949257.655207:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949257.655210:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949257.655212:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:3.0:1784949257.655215:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949257.655217:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949257.655218:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000046cb422a. 00000100:00000001:3.0:1784949257.655221:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949257.655223:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949257.655225:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949257.655230:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949257.655232:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949257.655234:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949257.655237:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949257.696000:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306197035, at 4306197036 00000100:00000040:0.0:1784949257.696013:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306197035, at 4306197036 00000100:00000010:1.0:1784949257.696015:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00000100:00000040:3.0:1784949257.696030:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306197035, at 4306197036 00000100:00000010:0.0:1784949257.696032:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00000100:00000040:2.0:1784949257.696043:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306197035, at 4306197036 00000100:00000010:3.0:1784949257.696048:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000ef8f137f. 00000100:00000010:2.0:1784949257.696059:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e19b789f. 00010000:00000001:2.0:1784949258.484038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949258.484048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949258.484056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949258.484058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949258.484188:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949258.484195:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949258.484199:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949258.676037:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949258.676054:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949258.676057:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949258.676064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949258.676069:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949258.676074:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949258.676077:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949259.087290:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949259.087324:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949259.087333:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949259.087336:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949259.508041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949259.508051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949259.508062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949259.508065:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949259.508156:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949259.508163:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949259.508168:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949259.700090:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949259.700106:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949259.700109:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949259.700116:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949259.700120:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949259.700125:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949259.700129:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949259.892048:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:2.0:1784949259.892055:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:2.0:1784949259.892064:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949259.892071:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949259.892080:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11532 health :900 00000400:00000200:2.0:1784949259.892086:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949259.892091:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:2.0:1784949259.892097:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:1.0:1784949259.892167:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949259.892179:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:1.0:1784949259.892188:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949259.892190:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949259.892193:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000400:00000010:1.0:1784949259.892197:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000400:00000200:1.0:1784949259.892203:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949259.892207:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:1.0:1784949259.892212:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000cdf2ee22. 00000400:00000200:1.0:1784949259.892219:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000400:00000200:1.0:1784949259.892222:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000400:00000200:1.0:1784949259.892227:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00010000:00000001:2.0:1784949260.532044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949260.532053:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949260.532061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949260.532063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949260.532135:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949260.532142:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949260.532147:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949260.724083:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949260.724099:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949260.724102:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949260.724107:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949260.724110:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949260.724114:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949260.724118:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949260.916108:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949260.916116:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000f1e3dc5f. 00000400:00000010:3.0:1784949260.916127:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000072ce5ef0. 00000400:00000010:3.0:1784949260.916134:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000400:00000010:3.0:1784949260.916144:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000325e9e7. 00000400:00000200:3.0:1784949260.916150:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949260.916168:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949260.916177:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949260.916182:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3802:3802:242:4294967295] 192.168.202.137@tcp LPNI seq info [2437:2437:8:4294967295] 00000400:00000200:3.0:1784949260.916191:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949260.916200:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949260.916207:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949260.916213:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000800:00000200:3.0:1784949260.916221:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949260.916232:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949260.916237:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949260.916264:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11534 health :900 00000800:00000200:0.0:1784949260.916449:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949260.916457:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:0.0:1784949260.916468:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949260.916480:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:2.0:1784949260.916484:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949260.916488:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949260.916495:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949260.916502:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949260.916509:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949260.916516:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949260.916530:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949260.916549:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949260.916560:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949260.916572:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949260.916578:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3803:3803:242:4294967295] 192.168.202.137@tcp LPNI seq info [2438:2438:8:4294967295] 00000400:00000200:2.0:1784949260.916589:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949260.916596:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949260.916601:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000800:00000200:2.0:1784949260.916608:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949260.916615:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949260.916619:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:2.0:1784949260.916739:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949260.916745:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000800:00000001:3.0:1784949260.916751:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949260.916753:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949260.916765:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949260.916768:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949260.916769:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949260.916786:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949260.916810:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949260.916816:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949260.916827:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949260.916843:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x93f9 00000800:00000001:0.0:1784949260.916850:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949260.916859:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949260.916862:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949260.916866:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949260.916873:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:0.0:1784949260.916877:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000400:00000200:0.0:1784949260.916880:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949260.916883:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000f1e3dc5f. 00000400:00000010:0.0:1784949260.916887:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000072ce5ef0. 00000800:00000001:0.0:1784949260.916894:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949260.916897:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949261.556034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949261.556043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949261.556051:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949261.556055:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949261.556058:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949261.556061:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949261.556065:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949261.556068:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949261.556070:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949261.556073:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949261.556205:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949261.556212:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949261.556216:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949261.748031:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949261.748046:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949261.748048:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949261.748054:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949261.748057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949261.748061:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949261.748065:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949262.580030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949262.580040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949262.580048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949262.580049:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949262.580159:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949262.580166:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949262.580170:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949262.772034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949262.772042:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949262.772060:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949237/real 1784949237] req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949262 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949262.772084:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949262.772096:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000001:2.0:1784949262.772097:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949262.772102:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949262.772106:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000acd175a6. 00000100:00000010:0.0:1784949262.772107:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b83ea808. 00000020:00000040:0.0:1784949262.772117:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000400:00000200:2.0:1784949262.772119:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:2.0:1784949262.772126:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000100:00000001:0.0:1784949262.772128:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949262.772134:0:337118:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1784949262.772135:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1784949262.772145:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949262 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949262.772146:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949262.772164:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000200:2.0:1784949262.772167:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949262 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949262.772176:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.772180:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949262.772185:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.772189:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1784949262.772191:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949262.772194:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949262.772211:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949262 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949262.772222:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949262.772227:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.772232:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949262.772240:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b83ea808] to pc [ptlrpcd_00_03+3] req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949262.772246:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949262.772252:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.772260:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949262.772265:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949262.772269:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949262.772277:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949262 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949262.772287:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949262.772291:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.772293:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949262.772294:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949262.772297:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.772298:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949262.772306:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949262.772307:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949262.772318:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949262.772322:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949262.772327:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949262.772331:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000050bfb6db. 00000100:00080000:2.0:1784949262.772332:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000040:0.0:1784949262.772337:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949262.772343:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949262.772346:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949262.772349:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949262.772353:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005a1c05a3. 00000100:00000001:3.0:1784949262.772357:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949262.772358:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949262.772364:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:3.0:1784949262.772369:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949262.772372:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949262.772377:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949262.772378:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949262.772387:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949262 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949262.772394:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949262.772412:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949262.772422:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.772429:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949262.772436:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949262.772451:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050bfb6db] to pc [ptlrpcd_00_00+0] req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949262.772465:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289683840:192.168.202.137@tcp:8: 00000100:00000001:3.0:1784949262.772466:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949262.772475:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949262.772489:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949262.772490:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949262.772496:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949262.772498:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784949262.772506:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949262.772508:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000bab47beb x1871651289683840/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949262 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949262.772516:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289685120:0@lo:400: 00000100:00000001:0.0:1784949262.772520:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949262.772533:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11539) 00000100:00000001:3.0:1784949262.772537:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949262.772548:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1784949262.772550:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 02000000:00000001:3.0:1784949262.772551:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949262.772556:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949262.772559:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949262.772562:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784949262.772565:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949262.772566:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949262.772569:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1784949262.772573:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1784949262.772575:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:3.0:1784949262.772580:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000378c7327. 00000100:00000001:0.0:1784949262.772583:0:335217:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1784949262.772584:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:1.0:1784949262.772588:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949262.772591:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949262.772593:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000077ad1613. 00000100:00080000:0.0:1784949262.772598:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11509 00000400:00000010:3.0:1784949262.772609:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a23805e4. 02000000:00000001:2.0:1784949262.772611:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949262.772620:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:0.0:1784949262.772621:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000400:00000010:3.0:1784949262.772627:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000020:00000040:2.0:1784949262.772628:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00080000:0.0:1784949262.772637:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11508 00000100:00000001:1.0:1784949262.772639:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1784949262.772641:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000200:3.0:1784949262.772646:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289685120, portal 25 00000100:00000001:1.0:1784949262.772648:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949262.772649:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000efbd7ee7. 00000400:00000200:0.0:1784949262.772654:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:3.0:1784949262.772658:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1784949262.772662:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949262.772664:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1784949262.772666:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469561856 : -115544239989760 : ffff96e9c29e9c00) 00000100:00000040:0.0:1784949262.772670:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949262.772673:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949262.772674:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1784949262.772681:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1784949262.772682:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000bab47beb. 02000000:00000001:1.0:1784949262.772683:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949262.772685:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949262.772693:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949262.772698:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00100000:1.0:1784949262.772704:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000050bfb6db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289685248:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949262.772705:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000001:2.0:1784949262.772706:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949262.772721:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949262.772722:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949262.772725:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1784949262.772729:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:1.0:1784949262.772736:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949262.772737:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949262.772738:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1784949262.772739:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1784949262.772744:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.772747:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949262.772749:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000040:3.0:1784949262.772752:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784949262.772755:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949262.772760:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784949262.772762:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:3.0:1784949262.772765:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000100:00000001:2.0:1784949262.772767:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949262.772768:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000b241612. 00000020:00000001:0.0:1784949262.772770:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:1.0:1784949262.772810:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949262.772819:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289685120, offset 0 00000100:00000040:0.0:1784949262.772822:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000400:00000010:1.0:1784949262.772822:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000001:0.0:1784949262.772834:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949262.772838:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784949262.772840:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000100:00000040:0.0:1784949262.772842:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000200:3.0:1784949262.772846:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949262.772847:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.772853:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:0.0:1784949262.772860:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000200:1.0:1784949262.772864:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289685248, portal 10 00000100:00080000:0.0:1784949262.772870:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000001:1.0:1784949262.772873:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949262.772881:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469557376 : -115544239994240 : ffff96e9c29e8a80) 00000400:00000200:3.0:1784949262.772883:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00080000:0.0:1784949262.772884:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000001:0.0:1784949262.772899:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949262.772951:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949262.772963:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1784949262.772969:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643dc80 02000000:00000001:0.0:1784949262.772976:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949262.772983:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949262.772985:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949262.772993:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8a71 [2] + 0 00000100:00000001:0.0:1784949262.772995:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1784949262.772996:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949262.773005:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949262.773006:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d3f471e1. 00000400:00000010:1.0:1784949262.773008:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000100:00000010:0.0:1784949262.773016:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000021c72e86. 00000400:00000200:3.0:1784949262.773020:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000a1b701f2 00000020:00000040:0.0:1784949262.773023:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:1.0:1784949262.773030:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289685248, offset 0 02000000:00000001:0.0:1784949262.773032:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1784949262.773037:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:0.0:1784949262.773038:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949262.773050:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949262.773051:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1784949262.773059:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784949262.773060:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949262.773067:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949262.773075:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 02000000:00000010:0.0:1784949262.773076:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000000b08bf97. 00000100:00000001:0.0:1784949262.773087:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949262.773088:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949262.773101:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00100000:0.0:1784949262.773106:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784949262.773113:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949262.773125:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:0.0:1784949262.773139:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1784949262.773139:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3804:3804:242:4294967295] 192.168.202.137@tcp LPNI seq info [2439:2439:8:4294967295] 00000100:00000001:0.0:1784949262.773144:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949262.773156:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021c72e86] to pc [ptlrpcd_rcv+-1] req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949262.773177:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949262.773196:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949262.773198:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a1b701f2 00000100:00000001:0.0:1784949262.773203:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949262.773205:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000a1b701f2. 00000800:00000010:1.0:1784949262.773206:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004b264730. 00000100:00000040:0.0:1784949262.773213:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11539) 00000100:00000001:3.0:1784949262.773221:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:1.0:1784949262.773228:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:3.0:1784949262.773229:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949262.773242:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000095b72665 x1871651289685120 msgsize 224 00000800:00000200:1.0:1784949262.773251:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:3.0:1784949262.773263:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000200:1.0:1784949262.773264:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004b264730 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949262.773276:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949262.773277:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949262.773287:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949262.773291:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.773294:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949262.773301:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2235 usecs (3 jiffies) 00000100:00000001:3.0:1784949262.773309:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949262.773310:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306202116 00000100:00000040:2.0:1784949262.773312:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949262.773325:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000400:00000200:3.0:1784949262.773331:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000100:00000001:1.0:1784949262.773340:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000400:00000010:3.0:1784949262.773342:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000100:00000001:1.0:1784949262.773358:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306202116 00000100:00000001:3.0:1784949262.773360:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949262.773361:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949262.773368:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949262.773372:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 02000000:00000001:2.0:1784949262.773374:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000200:3.0:1784949262.773380:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949262.773383:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949262.773385:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 02000000:00000001:2.0:1784949262.773395:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949262.773397:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306202116 00000100:00000010:1.0:1784949262.773408:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00100000:2.0:1784949262.773410:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000021c72e86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289685376:1.2.3.4@tcp:8: 00000100:00000001:1.0:1784949262.773419:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949262.773431:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949262.773433:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949262.773435:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306202116 00000100:00000001:2.0:1784949262.773436:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949262.773441:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:3.0:1784949262.773446:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949262.773447:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:2.0:1784949262.773449:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949262.773451:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004b264730. 00000100:00000001:1.0:1784949262.773461:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949262.773463:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949262.773466:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949262.773472:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949262.773478:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949262.773483:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949262.773488:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949262.773491:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949262.773496:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.773499:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.773500:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949262.773504:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949262.773506:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000100:00000001:3.0:1784949262.773512:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:0.0:1784949262.773514:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000100:00000001:2.0:1784949262.773515:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949262.773521:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949262.773522:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949262.773526:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949262.773527:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949262.773530:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949262.773531:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.773537:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306202155 00000100:00000200:0.0:1784949262.773539:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949262.773543:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949262.773545:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949262.773552:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949262.773559:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.773560:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306202116 00000100:00000001:0.0:1784949262.773567:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949262.773570:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.773578:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306202155 00000800:00000001:2.0:1784949262.773579:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949262.773580:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949262.773594:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306202116 00000100:00000001:0.0:1784949262.773597:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949262.773599:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949262.773606:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949262.773607:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306202155 00000800:00000001:2.0:1784949262.773608:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949262.773620:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949262.773626:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306202116 00000100:00000001:1.0:1784949262.773630:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949262.773638:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289685120 00000100:00000001:3.0:1784949262.773642:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306202155 00000400:00000200:2.0:1784949262.773645:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000001:1.0:1784949262.773646:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949262.773651:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306202116 00000100:00000001:1.0:1784949262.773654:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949262.773661:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.773664:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949262.773669:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.773674:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949262.773675:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643dd00 00000100:00100000:1.0:1784949262.773680:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289685120 00000100:00000001:3.0:1784949262.773683:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949262.773689:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949262.773691:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949262.773694:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 177776 00000020:00000040:1.0:1784949262.773696:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949262.773703:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1784949262.773704:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000001:2.0:1784949262.773709:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949262.773711:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000040:1.0:1784949262.773713:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000800:00000001:2.0:1784949262.773721:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949262.773724:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949262.773725:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949262.773728:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949262.773732:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1784949262.773734:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949262.773739:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949262.773745:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949262.773747:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949262.773749:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.773758:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000020:00000010:1.0:1784949262.773759:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000200:2.0:1784949262.773765:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784949262.773767:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949262.773797:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000010:1.0:1784949262.773809:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00000020:00000010:1.0:1784949262.773815:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000080b4bf82. 00000020:00000010:1.0:1784949262.773821:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949262.773832:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000010:2.0:1784949262.773835:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f790967e. 00000100:00000001:1.0:1784949262.773836:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949262.773843:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:2.0:1784949262.773846:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000f790967e x1871651289685248 msgsize 224 00000100:00000001:1.0:1784949262.773847:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949262.773861:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949262.773863:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949262.773878:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.773887:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949262.773893:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949262.773898:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:2.0:1784949262.773948:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:1.0:1784949262.773950:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 813 00000100:00000001:0.0:1784949262.773955:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:2.0:1784949262.773959:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949262.773960:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00100000:0.0:1784949262.773962:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289685248 00000100:00000001:1.0:1784949262.773972:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818128976 : -115538891422640 : ffff96eb016b4450) 02000000:00000001:0.0:1784949262.773973:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949262.773981:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949262.773985:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949262.773987:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000095b72665 x1871651289685120/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:168/0 lens 224/0 e 0 to 0 dl 1784949273 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949262.773992:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949262.774003:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289685248 00000020:00000001:0.0:1784949262.774011:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1784949262.774013:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000040:0.0:1784949262.774016:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000010:1.0:1784949262.774021:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000020:00000001:0.0:1784949262.774022:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949262.774030:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:1.0:1784949262.774031:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949262.774040:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000010:1.0:1784949262.774041:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000001:0.0:1784949262.774052:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00100000:1.0:1784949262.774060:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000095b72665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289685120:12345-0@lo:400: 00000100:00000001:0.0:1784949262.774063:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:1.0:1784949262.774071:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289685120 00000020:00000010:0.0:1784949262.774073:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000020:00000001:1.0:1784949262.774076:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:0.0:1784949262.774081:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949262.774085:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000010:0.0:1784949262.774087:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000020:00000001:1.0:1784949262.774091:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949262.774096:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000010:0.0:1784949262.774097:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000020:00000001:1.0:1784949262.774100:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000010:0.0:1784949262.774107:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002eb61d28. 00000020:00000001:1.0:1784949262.774109:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949262.774115:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949262.774118:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002ffa19fa. 00000020:00000001:1.0:1784949262.774120:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949262.774127:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949262.774130:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949262.774134:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:0.0:1784949262.774135:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949262.774141:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949262.774142:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784949262.774148:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949262.774149:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949262.774157:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949262.774159:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000579609a4. 00000100:00000001:0.0:1784949262.774165:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000001:1.0:1784949262.774166:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.774173:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949262.774174:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.774179:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949262.774181:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949262.774190:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289685120 00010000:00000001:1.0:1784949262.774194:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949262.774199:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000095b72665 x1871651289685120/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:168/0 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949262.774202:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949262.774217:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949262.774221:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00010000:00000001:1.0:1784949262.774221:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.774228:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00001000:1.0:1784949262.774230:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=134 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949262.774238:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1784949262.774240:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 789 00000100:00000040:1.0:1784949262.774247:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:0.0:1784949262.774250:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949262.774255:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:0.0:1784949262.774257:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469575552 : -115544239976064 : ffff96e9c29ed180) 02000000:00000001:1.0:1784949262.774263:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949262.774268:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949262.774272:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f790967e x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:168/0 lens 224/0 e 0 to 0 dl 1784949273 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949262.774275:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949262.774287:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949262.774295:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000200:1.0:1784949262.774304:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289685120, offset 224 00000400:00000200:1.0:1784949262.774321:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949262.774328:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949262.774335:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000680019f0. 00000400:00000200:1.0:1784949262.774338:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949262.774342:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949262.774347:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000680019f0. 00000400:00000200:1.0:1784949262.774360:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643dc80 00000100:00100000:0.0:1784949262.774362:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f790967e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289685248:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949262.774374:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289685248 00000400:00000200:1.0:1784949262.774376:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x93fd [1] + 224 00000020:00000001:0.0:1784949262.774380:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949262.774388:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:1.0:1784949262.774391:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949262.774392:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949262.774401:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:1.0:1784949262.774406:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:0.0:1784949262.774407:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949262.774415:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949262.774416:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1784949262.774420:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949262.774427:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949262.774429:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949262.774434:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949262.774439:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949262.774446:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949262.774453:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:1.0:1784949262.774458:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949262.774460:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949262.774468:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000be80dc96. 02000000:00000001:0.0:1784949262.774479:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.774482:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949262.774485:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949262.774488:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289685248 00010000:00000001:0.0:1784949262.774491:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949262.774496:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000200:0.0:1784949262.774500:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f790967e x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:168/0 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949262.774509:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00010000:00000001:0.0:1784949262.774516:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000010:1.0:1784949262.774517:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00010000:00000001:0.0:1784949262.774524:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949262.774526:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949262.774532:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00001000:0.0:1784949262.774534:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=129 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949262.774539:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.774541:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949262.774547:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000040:0.0:1784949262.774552:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949262.774554:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949262.774560:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:1.0:1784949262.774562:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949262.774570:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:1.0:1784949262.774571:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 02000000:00000001:0.0:1784949262.774577:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949262.774584:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949262.774584:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000095b72665 x1871651289685120/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:168/0 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949262.774599:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949262.774607:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdaa2431. 00000100:00000001:3.0:1784949262.774612:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949262.774617:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289685248, offset 224 00000100:00000001:3.0:1784949262.774620:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1784949262.774621:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000095b72665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289685120:12345-0@lo:400: Request processed in 600us (1370us total) trans 0 rc 0/0 00000400:00000200:0.0:1784949262.774635:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784949262.774645:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 813 00000400:00000001:3.0:1784949262.774646:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949262.774655:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1784949262.774656:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000040:1.0:1784949262.774657:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000010:3.0:1784949262.774662:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a23805e4. 00000100:00000001:1.0:1784949262.774668:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949262.774676:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:0.0:1784949262.774677:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3805:3805:242:4294967295] 192.168.202.137@tcp LPNI seq info [2440:2440:8:4294967295] 00000400:00000200:3.0:1784949262.774678:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00000020:00000010:1.0:1784949262.774686:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000080b4bf82. 00000400:00000010:3.0:1784949262.774687:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000020:00000010:1.0:1784949262.774700:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949262.774703:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1784949262.774711:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:1.0:1784949262.774713:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000100:00000200:3.0:1784949262.774719:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949262.774725:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000800:00000200:0.0:1784949262.774731:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949262.774737:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949262.774745:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000579609a4. 00000800:00000010:0.0:1784949262.774746:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002a753d9d. 00000100:00000200:3.0:1784949262.774757:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949262.774759:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949262.774767:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949262.774768:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:1.0:1784949262.774805:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000800:00000200:0.0:1784949262.774814:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784949262.774815:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000010:1.0:1784949262.774822:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000026bb7060. 00000100:00000001:3.0:1784949262.774822:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949262.774823:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000002a753d9d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949262.774836:0:337122:0:(client.c:1580:after_reply()) Process entered 00000400:00000010:1.0:1784949262.774841:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006a025321. 02000000:00000001:3.0:1784949262.774843:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949262.774849:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949262.774853:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.774854:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949262.774864:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949262.774867:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00001000:3.0:1784949262.774873:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=134 v=5 (5 5 5 5) 00010000:00000001:0.0:1784949262.774874:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949262.774880:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949262.774885:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=134 v=5 (1 1 1 1) 00000020:00000001:0.0:1784949262.774886:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784949262.774894:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000040:0.0:1784949262.774898:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f790967e x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:168/0 lens 224/224 e 0 to 0 dl 1784949273 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949262.774900:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949262.774942:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949262.774945:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949262.774949:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949262.774965:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949262.774977:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f790967e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289685248:12345-192.168.202.137@tcp:400: Request processed in 642us (1124us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949262.774998:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949262.775004:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:0.0:1784949262.775006:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 789 00000100:00000001:3.0:1784949262.775008:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949262.775013:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000040:3.0:1784949262.775018:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949262.775021:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949262.775029:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949262.775036:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002eb61d28. 00000100:00100000:3.0:1784949262.775041:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289685120:0@lo:400: 00000020:00000010:0.0:1784949262.775045:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002ffa19fa. 00000100:00000001:3.0:1784949262.775053:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949262.775054:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000100:00000001:3.0:1784949262.775061:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:0.0:1784949262.775067:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000040:3.0:1784949262.775071:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b83ea808 x1871651289685120/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949262.775075:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949262.775082:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949262.775093:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002a753d9d. 00000100:00000001:3.0:1784949262.775095:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:1.0:1784949262.775099:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949262.775102:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:2.0:1784949262.775106:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:3.0:1784949262.775107:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000378c7327. 00000800:00000001:1.0:1784949262.775114:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949262.775120:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000001:1.0:1784949262.775121:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949262.775128:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000800:00000001:1.0:1784949262.775132:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949262.775132:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000040:3.0:1784949262.775133:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949262.775146:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:2.0:1784949262.775151:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdaa2431 02000000:00000010:3.0:1784949262.775153:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 00000400:00000200:1.0:1784949262.775154:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:2.0:1784949262.775163:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdaa2431. 02000000:00000001:3.0:1784949262.775166:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949262.775174:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:1.0:1784949262.775176:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643dd00 00000100:00000001:2.0:1784949262.775179:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000010:3.0:1784949262.775180:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b83ea808. 00000100:00000001:2.0:1784949262.775190:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949262.775191:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9405 [1] + 224 00000100:00000001:3.0:1784949262.775194:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949262.775200:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949262.775206:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949262.775207:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949262.775220:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949262.775223:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949262.775225:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949262.775228:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784949262.775232:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949262.775236:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949262.775244:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949262.775252:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949262.775257:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949262.775270:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949262.775295:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949262.775303:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949262.775306:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.775347:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949262.775352:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949262.775358:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949262.775360:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949262.775363:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000400:00000200:2.0:1784949262.775369:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:2.0:1784949262.775371:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000100:00000001:2.0:1784949262.775374:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949262.775382:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949262.775395:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949262.775405:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949262.775407:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.775410:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949262.775412:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949262.775414:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949262.775417:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949262.775419:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949262.775426:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=129 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949262.775430:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=129 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949262.775433:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949262.775435:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949262.775437:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949262.775441:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949262.775451:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.775455:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949262.775457:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949262.775459:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949262.775461:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949262.775464:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949262.775472:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949262.775474:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949262.775476:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949262.775479:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949262.775487:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000050bfb6db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289685248:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949262.775493:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.775495:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949262.775497:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000050bfb6db x1871651289685248/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949278 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949262.775506:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949262.775508:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949262.775511:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000b241612. 02000000:00000001:2.0:1784949262.775515:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949262.775516:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949262.775518:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949262.775521:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949262.775523:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005a1c05a3. 02000000:00000001:2.0:1784949262.775525:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949262.775527:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949262.775529:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000050bfb6db. 00000100:00000001:2.0:1784949262.775533:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949262.775534:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.775536:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949262.775542:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949262.775544:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949262.775546:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949262.775549:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949262.816019:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306202155, at 4306202156 00000100:00000010:0.0:1784949262.816034:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000040:1.0:1784949262.816049:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306202155, at 4306202156 00000100:00000040:2.0:1784949262.816050:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306202155, at 4306202156 00000100:00000040:3.0:1784949262.816060:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306202155, at 4306202156 00000100:00000010:2.0:1784949262.816070:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:1.0:1784949262.816075:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000010:3.0:1784949262.816079:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00000800:00000100:2.0:1784949262.964077:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.2@tcp 00000800:00000010:2.0:1784949262.964085:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:2.0:1784949262.964093:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949262.964100:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949262.964107:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11536 health :900 00000400:00000200:2.0:1784949262.964113:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949262.964119:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:2.0:1784949262.964124:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000800:00000100:2.0:1784949262.964153:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000800:00000010:2.0:1784949262.964157:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000007a3f8609. 00000400:00000200:2.0:1784949262.964161:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949262.964166:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949262.964170:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949262.964175:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:2.0:1784949262.964179:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:1.0:1784949262.964203:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949262.964221:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000001:1.0:1784949262.964233:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949262.964237:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949262.964241:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:1.0:1784949262.964246:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:1.0:1784949262.964254:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949262.964261:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:1.0:1784949262.964269:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000d531934e. 00000400:00000200:1.0:1784949262.964284:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-110 00000400:00000200:1.0:1784949262.964290:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -110 00000400:00000200:1.0:1784949262.964297:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:1.0:1784949262.964303:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000001:1.0:1784949262.964308:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949262.964310:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949262.964312:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:1.0:1784949262.964314:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000400:00000200:1.0:1784949262.964317:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949262.964321:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:1.0:1784949262.964324:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000eae26e4. 00000400:00000200:1.0:1784949262.964330:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000200:1.0:1784949262.964333:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:1.0:1784949262.964336:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00010000:00000001:2.0:1784949263.604035:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949263.604044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949263.604052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949263.604053:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949263.604194:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949263.604201:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949263.604205:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949263.795980:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949263.795987:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949263.796004:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949262/real 0] req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949263 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949263.796024:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.796028:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949263.796031:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949263.796036:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949263 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949263.796049:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.796061:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949263.796064:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.796068:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949263.796072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949263.796074:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949263.796078:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949263 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949263.796088:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949263.796092:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.796094:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949263.796095:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.796098:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949263.796100:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949263.796105:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949263.796134:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:2.0:1784949263.796138:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949263.796143:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949263 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949263.796154:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000021c72e86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289685376:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949263.796163:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.796166:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949263.796171:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000021c72e86 x1871651289685376/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949263 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949263.796181:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1784949263.796186:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949263.796188:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784949263.796192:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949263.796198:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000000b08bf97. 02000000:00000001:2.0:1784949263.796207:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949263.796210:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949263.796213:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000021c72e86. 00000100:00000001:2.0:1784949263.796221:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949263.796223:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.796227:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.796232:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949263.796235:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.796241:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949263.796243:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.796245:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949263.796255:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949263.796265:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f230fea2. 00000020:00000040:0.0:1784949263.796272:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949263.796279:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949263.796283:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949263.796290:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949263.796301:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000034aacccc. 00000100:00000001:0.0:1784949263.796310:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.796320:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949263.796335:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949263.796339:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.796347:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f230fea2] to pc [ptlrpcd_00_01+1] req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949263.796395:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949263.796400:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949263.796406:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949263.796410:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949263.796413:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949263.796420:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949263.796428:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11509 00000100:00000001:2.0:1784949263.796430:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949263.796439:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949263.796444:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784949263.796447:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:3.0:1784949263.796455:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949263.796456:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784949263.796464:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11534 00000100:00000001:3.0:1784949263.796466:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.796485:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784949263.796489:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:2.0:1784949263.796491:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949263.796505:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1784949263.796505:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949263.796509:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949263.796518:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.796523:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00100000:2.0:1784949263.796527:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f230fea2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289685504:0@lo:400: 00000100:00000001:0.0:1784949263.796537:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949263.796539:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949263.796551:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784949263.796552:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:2.0:1784949263.796557:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949263.796563:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949263.796564:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1784949263.796574:0:335217:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000010:2.0:1784949263.796575:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000032fe57fd. 00000020:00000040:0.0:1784949263.796579:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:2.0:1784949263.796581:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949263.796586:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:2.0:1784949263.796594:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000179aef9d. 00000020:00000040:0.0:1784949263.796595:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:0.0:1784949263.796605:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784949263.796610:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:2.0:1784949263.796613:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000100:00000040:0.0:1784949263.796616:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949263.796623:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1784949263.796628:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289685504, portal 25 00000100:00000040:0.0:1784949263.796629:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949263.796642:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:2.0:1784949263.796642:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949263.796648:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469555584 : -115544239996032 : ffff96e9c29e8380) 00000020:00000040:0.0:1784949263.796650:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:0.0:1784949263.796661:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000040:2.0:1784949263.796665:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784949263.796674:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:0.0:1784949263.796685:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.796687:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949263.796691:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1784949263.796697:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784949263.796703:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949263.796704:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000200:2.0:1784949263.796717:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289685504, offset 0 02000000:00000001:0.0:1784949263.796718:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949263.796730:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784949263.796735:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949263.796735:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000010:0.0:1784949263.796746:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ca646268. 00000020:00000040:0.0:1784949263.796753:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784949263.796758:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949263.796794:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949263.796796:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949263.796805:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949263.796810:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949263.796814:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949263.796821:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643de00 02000000:00000010:0.0:1784949263.796825:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000003e4182b6. 00000100:00000001:0.0:1784949263.796835:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949263.796835:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8a89 [2] + 0 00000400:00000010:2.0:1784949263.796849:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000007a72fba7. 00000100:00100000:0.0:1784949263.796855:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1784949263.796860:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000005df8d94f 00000100:00000001:0.0:1784949263.796873:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1784949263.796875:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949263.796877:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.796892:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca646268] to pc [ptlrpcd_rcv+-1] req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949263.796893:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:2.0:1784949263.796951:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005df8d94f 00000400:00000010:2.0:1784949263.796960:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000005df8d94f. 00000100:00000001:2.0:1784949263.796970:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949263.796973:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784949263.796976:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949263.796979:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f911f57b x1871651289685504 msgsize 224 00000100:00000040:0.0:1784949263.796985:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949263.796996:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00100000:2.0:1784949263.796998:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949263.797004:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000010:0.0:1784949263.797005:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f84ecbdd. 00000100:00000001:1.0:1784949263.797006:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784949263.797017:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949263.797017:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949263.797021:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949263.797024:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949263.797028:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949263.797033:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949263.797036:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949263.797037:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000100:00000040:1.0:1784949263.797039:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949263.797046:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 02000000:00000010:0.0:1784949263.797047:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000009ff40a64. 00000100:00000001:0.0:1784949263.797060:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.797062:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949263.797072:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:2.0:1784949263.797074:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949263.797078:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.797080:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1784949263.797091:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949263.797099:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949263.797112:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949263.797119:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949263.797129:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289685504 00000100:00100000:1.0:1784949263.797131:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289685632:192.168.202.137@tcp:8: 00000100:00000001:2.0:1784949263.797134:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784949263.797140:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:2.0:1784949263.797147:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949263.797148:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949263.797151:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949263.797153:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949263.797155:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949263.797159:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949263.797163:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949263.797167:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949263.797169:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949263.797180:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1784949263.797183:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289685504 00000100:00000040:0.0:1784949263.797189:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f84ecbdd] to pc [ptlrpcd_00_02+2] req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784949263.797192:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000079de2679. 00000020:00000001:3.0:1784949263.797194:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949263.797195:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949263.797202:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 02000000:00000001:1.0:1784949263.797204:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.797206:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1784949263.797212:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949263.797219:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949263.797220:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000035db9be3. 00000020:00000040:3.0:1784949263.797221:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949263.797233:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306203179 00000400:00000010:1.0:1784949263.797237:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000010:2.0:1784949263.797244:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e19b789f. 00000100:00000200:1.0:1784949263.797247:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289685632, portal 4 00000100:00000001:0.0:1784949263.797251:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.797256:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949263.797260:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1784949263.797262:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949263.797264:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469564544 : -115544239987072 : ffff96e9c29ea680) 00000100:00000040:0.0:1784949263.797266:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11540) 00000020:00000001:3.0:1784949263.797278:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949263.797279:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306203179 00000100:00000040:1.0:1784949263.797288:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949263.797292:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:2.0:1784949263.797293:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000ef8f137f. 00000100:00000001:0.0:1784949263.797294:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949263.797304:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1784949263.797305:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000cbd03aa3. 00000100:00000001:2.0:1784949263.797307:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949263.797311:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949263.797321:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949263.797323:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949263.797325:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306203179 00000100:00000040:0.0:1784949263.797331:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949263.797339:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000cbd03aa3. 00000100:00000040:1.0:1784949263.797340:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000010:2.0:1784949263.797342:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000400:00000010:1.0:1784949263.797353:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000100:00000001:2.0:1784949263.797356:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:3.0:1784949263.797360:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000378c7327. 00000100:00000200:1.0:1784949263.797364:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289685632, offset 0 00000100:00000001:2.0:1784949263.797367:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306203179 00000020:00000010:3.0:1784949263.797375:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000001ae5b8eb. 00000100:00000010:2.0:1784949263.797383:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001cae17e9. 00000020:00000010:3.0:1784949263.797391:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000200:1.0:1784949263.797393:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949263.797397:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949263.797412:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949263.797413:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1784949263.797418:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:1.0:1784949263.797421:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784949263.797423:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949263.797425:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949263.797428:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949263.797439:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949263.797440:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949263.797441:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784949263.797443:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784949263.797446:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.797451:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949263.797453:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949263.797460:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:2.0:1784949263.797461:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949263.797462:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949263.797468:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.797470:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:1.0:1784949263.797477:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784949263.797485:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949263.797486:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.797492:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949263.797494:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00100000:0.0:1784949263.797495:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f84ecbdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289685760:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949263.797498:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.797508:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949263.797514:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3806:3806:244:4294967295] 192.168.202.137@tcp LPNI seq info [2441:2441:8:4294967295] 00000100:00000001:3.0:1784949263.797514:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949263.797518:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949263.797522:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:0.0:1784949263.797530:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1784949263.797537:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 814 02000000:00000001:0.0:1784949263.797540:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.797546:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949263.797549:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 02000000:00000001:0.0:1784949263.797552:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949263.797555:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784949263.797559:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949263.797562:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818136144 : -115538891415472 : ffff96eb016b6050) 00000100:00000001:2.0:1784949263.797563:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949263.797566:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 00000100:00000001:2.0:1784949263.797576:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949263.797577:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949263.797583:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f911f57b x1871651289685504/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:169/0 lens 224/0 e 0 to 0 dl 1784949274 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949263.797587:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949263.797595:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000948ad730. 00000800:00000010:1.0:1784949263.797600:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005c010cf6. 00000400:00000010:0.0:1784949263.797611:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000553902b5. 00000100:00000001:3.0:1784949263.797619:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:1.0:1784949263.797621:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784949263.797624:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289685760, portal 10 00000020:00000010:3.0:1784949263.797628:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000cbd03aa3. 00000100:00000001:0.0:1784949263.797631:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949263.797638:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469560064 : -115544239991552 : ffff96e9c29e9500) 00000100:00000001:3.0:1784949263.797639:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1784949263.797643:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000010:3.0:1784949263.797654:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000cbd03aa3. 00000800:00000200:1.0:1784949263.797655:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005c010cf6 type 1, nob 616 niov 1 nkiov 1 00000100:00000040:0.0:1784949263.797665:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949263.797677:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f911f57b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289685504:12345-0@lo:400: 00000100:00000001:0.0:1784949263.797689:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949263.797689:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949263.797692:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289685504 00000100:00000040:1.0:1784949263.797703:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000020:00000001:3.0:1784949263.797703:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000040:0.0:1784949263.797705:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949263.797714:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306203179 00000020:00000001:3.0:1784949263.797714:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:0.0:1784949263.797717:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000020:00000001:3.0:1784949263.797723:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.797727:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306203179 00000100:00000200:0.0:1784949263.797729:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289685760, offset 0 00000020:00000001:3.0:1784949263.797731:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949263.797737:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306203179 00000020:00000001:3.0:1784949263.797739:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784949263.797747:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306203179 00000400:00000200:0.0:1784949263.797749:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784949263.797752:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949263.797758:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306203179 00000020:00000001:3.0:1784949263.797760:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949263.797793:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306203179 00000020:00000001:3.0:1784949263.797797:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:0.0:1784949263.797802:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949263.797806:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306203179 00000020:00000001:3.0:1784949263.797807:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:0.0:1784949263.797813:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784949263.797815:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.797817:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306203179 00000020:00000001:3.0:1784949263.797824:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:0.0:1784949263.797826:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949263.797828:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949263.797832:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949263.797840:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949263.797841:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1784949263.797842:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949263.797853:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949263.797854:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000010:3.0:1784949263.797855:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c38711c7. 00000100:00000001:1.0:1784949263.797859:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949263.797868:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949263.797872:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3807:3807:243:4294967295] 192.168.202.137@tcp LPNI seq info [2442:2442:7:4294967295] 00000100:00000001:1.0:1784949263.797874:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949263.797876:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.797881:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:3.0:1784949263.797888:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.797891:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949263.797898:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000040:3.0:1784949263.797899:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289685504 00000100:00000001:1.0:1784949263.797940:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.797945:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949263.797954:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:0.0:1784949263.797956:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00010000:00000200:3.0:1784949263.797966:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f911f57b x1871651289685504/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:169/0 lens 224/224 e 0 to 0 dl 1784949274 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784949263.797978:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949263.797988:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:0.0:1784949263.797994:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000057c7ab4. 00000100:00000001:1.0:1784949263.797997:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949263.797998:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949263.798006:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949263.798013:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00001000:3.0:1784949263.798017:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=135 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949263.798026:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1784949263.798028:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000040:3.0:1784949263.798034:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000200:0.0:1784949263.798036:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000057c7ab4 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949263.798044:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949263.798051:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949263.798053:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949263.798058:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949263.798063:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949263.798066:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949263.798070:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000100:00000040:0.0:1784949263.798074:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2236 usecs (3 jiffies) 00000100:00000200:3.0:1784949263.798080:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289685504, offset 224 00000100:00000001:0.0:1784949263.798082:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306203141 00000400:00000200:3.0:1784949263.798100:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949263.798116:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784949263.798133:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643de00 00000400:00000200:3.0:1784949263.798144:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9419 [1] + 224 00000400:00000200:3.0:1784949263.798153:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949263.798161:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949263.798171:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949263.798179:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949263.798202:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949263.798221:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306203179 00000800:00000200:2.0:1784949263.798226:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949263.798227:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949263.798229:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306203141 00000100:00000001:1.0:1784949263.798238:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306203179 00000800:00000200:0.0:1784949263.798239:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949263.798240:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005c010cf6. 00000400:00000200:3.0:1784949263.798243:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00000100:00000001:1.0:1784949263.798249:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306203141 00000800:00000010:0.0:1784949263.798254:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000057c7ab4. 00000400:00000010:3.0:1784949263.798255:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000100:00000001:1.0:1784949263.798256:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306203179 00000400:00000200:2.0:1784949263.798258:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949263.798270:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306203141 00000400:00000200:0.0:1784949263.798273:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949263.798274:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949263.798277:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306203179 00000100:00000001:3.0:1784949263.798285:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949263.798286:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949263.798287:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949263.798298:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949263.798300:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949263.798301:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949263.798305:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000100:00000040:3.0:1784949263.798311:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949263.798312:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949263.798314:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00010000:00000001:3.0:1784949263.798324:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949263.798325:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.798328:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784949263.798330:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000020:00000001:3.0:1784949263.798332:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.798337:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:0.0:1784949263.798339:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000020:00000001:3.0:1784949263.798340:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000200:2.0:1784949263.798345:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949263.798346:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949263.798352:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1784949263.798357:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f911f57b x1871651289685504/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:169/0 lens 224/224 e 0 to 0 dl 1784949274 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949263.798358:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949263.798364:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:0.0:1784949263.798369:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949263.798373:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.798386:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949263.798408:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949263.798410:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949263.798425:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949263.798427:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949288 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949263.798429:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949263.798430:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f911f57b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289685504:12345-0@lo:400: Request processed in 801us (1439us total) trans 0 rc 0/0 00000100:00000040:0.0:1784949263.798441:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949263.798444:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949263.798453:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949263.798477:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 814 00000400:00000200:1.0:1784949263.798484:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:3.0:1784949263.798493:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949263.798506:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949263.798513:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:1.0:1784949263.798525:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643de80 00000020:00000010:3.0:1784949263.798526:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000001ae5b8eb. 00000100:00000001:0.0:1784949263.798529:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.798537:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949263.798541:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949263.798543:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949263.798544:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000100:1.0:1784949263.798552:0:337112:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289685632 offset 0 length 520: 4 00000020:00000010:3.0:1784949263.798556:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000378c7327. 00000400:00000200:1.0:1784949263.798560:0:337112:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000279db1a3 is in inconsistent state, don't perform health checking (-2, 0) 00000800:00000001:2.0:1784949263.798562:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000040:3.0:1784949263.798566:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000200:1.0:1784949263.798567:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000800:00000001:2.0:1784949263.798581:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949263.798582:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:3.0:1784949263.798587:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000c38711c7. 00000800:00000001:1.0:1784949263.798589:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949263.798594:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949263.798602:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000800:00000001:1.0:1784949263.798606:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949263.798608:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949263.798611:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949263.798620:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784949263.798623:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000800:00000001:1.0:1784949263.798630:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949263.798631:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949263.798636:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a23805e4. 00000100:00000001:0.0:1784949263.798643:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949263.798643:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:3.0:1784949263.798648:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000152a1f57. 00000400:00000001:0.0:1784949263.798655:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949263.798660:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949263.798665:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000179aef9d. 00000400:00000200:2.0:1784949263.798675:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643df00 00000400:00000200:0.0:1784949263.798676:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:0.0:1784949263.798682:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000100:00000001:0.0:1784949263.798692:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949263.798703:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949263.798710:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 178000 00000100:00000200:0.0:1784949263.798736:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949263.798739:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949263.798752:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949263.798755:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949263.798759:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949263.798762:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949263.798788:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949263.798791:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949263.798791:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949263.798797:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949263.798804:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00001000:0.0:1784949263.798807:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=135 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949263.798815:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=135 v=5 (1 1 1 1) 00000400:00000200:2.0:1784949263.798825:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949263.798826:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949263.798832:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949263.798838:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784949263.798842:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.798842:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784949263.798852:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949263.798852:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:0.0:1784949263.798866:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1784949263.798868:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d611c85c. 00000100:00000040:2.0:1784949263.798882:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000d611c85c x1871651289685760 msgsize 224 00000100:00000001:0.0:1784949263.798887:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949263.798898:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1784949263.798946:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949263.798962:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.798973:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784949263.798977:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949263.798990:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:0.0:1784949263.799002:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f230fea2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289685504:0@lo:400: 00000800:00000001:2.0:1784949263.799003:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949263.799013:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949263.799021:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289685760 00000100:00000001:0.0:1784949263.799025:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949263.799026:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949263.799030:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949263.799037:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949263.799038:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784949263.799048:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.799049:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f230fea2 x1871651289685504/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949263.799057:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289685760 00000020:00000001:1.0:1784949263.799065:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949263.799070:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784949263.799077:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1784949263.799077:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949263.799081:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:0.0:1784949263.799084:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:1.0:1784949263.799092:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000010:0.0:1784949263.799093:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000032fe57fd. 00000020:00000001:1.0:1784949263.799101:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:0.0:1784949263.799105:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1784949263.799108:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000001:0.0:1784949263.799111:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949263.799115:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000010:1.0:1784949263.799117:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000020:00000001:0.0:1784949263.799119:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:1.0:1784949263.799126:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1784949263.799127:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000034aacccc. 00000020:00000010:1.0:1784949263.799137:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 02000000:00000001:0.0:1784949263.799138:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949263.799142:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949263.799146:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f230fea2. 00000020:00000010:1.0:1784949263.799150:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004f865a97. 00000100:00000001:0.0:1784949263.799154:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000010:1.0:1784949263.799158:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000080b4bf82. 00000100:00000001:0.0:1784949263.799159:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949263.799169:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949263.799170:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949263.799176:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949263.799182:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949263.799183:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949263.799187:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949263.799190:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949263.799193:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.799197:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949263.799203:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949263.799210:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949263.799215:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949263.799218:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.799254:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949263.799272:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949263.799275:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949263.799286:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 790 00000100:00000040:1.0:1784949263.799292:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949263.799297:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469571072 : -115544239980544 : ffff96e9c29ec000) 00000100:00000040:1.0:1784949263.799307:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d611c85c x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:169/0 lens 224/0 e 0 to 0 dl 1784949274 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949263.799326:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949263.799330:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784949263.799333:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949263.799336:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00100000:1.0:1784949263.799342:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d611c85c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289685760:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949263.799347:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289685760 00000020:00000001:1.0:1784949263.799350:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949263.799354:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949263.799356:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949263.799359:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949263.799362:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949263.799365:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949263.799368:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949263.799370:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949263.799372:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949263.799374:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949263.799377:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949263.799381:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949263.799383:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949263.799388:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009925ca9f. 02000000:00000001:1.0:1784949263.799391:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.799393:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949263.799395:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949263.799398:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289685760 00010000:00000001:1.0:1784949263.799401:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949263.799405:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d611c85c x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:169/0 lens 224/224 e 0 to 0 dl 1784949274 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949263.799415:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949263.799417:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949263.799420:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=130 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949263.799425:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949263.799428:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949263.799430:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949263.799433:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949263.799435:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949263.799438:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949263.799441:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949263.799444:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000200:1.0:1784949263.799448:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289685760, offset 224 00000400:00000200:1.0:1784949263.799454:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949263.799463:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949263.799470:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3808:3808:244:4294967295] 192.168.202.137@tcp LPNI seq info [2443:2443:8:4294967295] 00000400:00000200:1.0:1784949263.799481:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949263.799487:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949263.799491:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fee8262a. 00000800:00000200:1.0:1784949263.799497:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949263.799503:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949263.799507:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fee8262a type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949263.799525:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949263.799529:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949263.799531:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949263.799534:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949263.799536:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949263.799540:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d611c85c x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:169/0 lens 224/224 e 0 to 0 dl 1784949274 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949263.799553:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d611c85c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289685760:12345-192.168.202.137@tcp:400: Request processed in 226us (669us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949263.799563:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 790 00000100:00000040:1.0:1784949263.799567:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949263.799570:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949263.799573:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949263.799576:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000080b4bf82. 00000020:00000010:1.0:1784949263.799582:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949263.799588:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004f865a97. 00000020:00000040:1.0:1784949263.799595:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949263.799602:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949263.799694:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949263.799700:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fee8262a. 00000400:00000200:0.0:1784949263.799706:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949263.799713:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949263.799719:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:0.0:1784949263.799723:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000800:00000001:1.0:1784949263.799728:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949263.799729:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949263.799736:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949263.799744:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949263.799749:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949263.799754:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949263.799785:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949263.799798:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643df00 00000400:00000200:1.0:1784949263.799806:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9429 [1] + 224 00000800:00000001:1.0:1784949263.799812:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949263.799822:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949263.799825:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949263.799830:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949263.799835:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949263.799840:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949263.799852:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949263.799885:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949263.799892:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949263.799895:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.799954:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949263.799959:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949263.799965:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949263.799967:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949263.799971:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000948ad730. 00000400:00000200:2.0:1784949263.799977:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000553902b5 00000400:00000010:2.0:1784949263.799979:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000553902b5. 00000100:00000001:2.0:1784949263.799983:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949263.799989:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949263.800000:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949263.800017:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949263.800019:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.800022:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949263.800024:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949263.800026:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949263.800028:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949263.800031:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949263.800038:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=130 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949263.800042:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=130 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949263.800045:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949263.800046:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949263.800049:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949263.800053:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949263.800061:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.800065:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949263.800066:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949263.800069:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949263.800070:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949263.800073:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949263.800082:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949263.800084:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949263.800086:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949263.800089:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949263.800098:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f84ecbdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289685760:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949263.800103:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.800105:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949263.800107:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f84ecbdd x1871651289685760/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949263.800122:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949263.800124:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949263.800127:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:2.0:1784949263.800131:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949263.800133:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949263.800134:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949263.800137:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949263.800139:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000009ff40a64. 02000000:00000001:2.0:1784949263.800142:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949263.800143:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949263.800145:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f84ecbdd. 00000100:00000001:2.0:1784949263.800149:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949263.800151:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.800153:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949263.800158:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949263.800159:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949263.800162:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949263.800164:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949263.839990:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306203179, at 4306203180 00000100:00000010:0.0:1784949263.840006:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e19b789f. 00000100:00000040:1.0:1784949263.840042:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306203179, at 4306203180 00000100:00000010:1.0:1784949263.840057:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000ef8f137f. 00000100:00000040:2.0:1784949263.840059:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306203179, at 4306203180 00000100:00000040:3.0:1784949263.840062:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306203179, at 4306203180 00000100:00000010:2.0:1784949263.840076:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00000100:00000010:3.0:1784949263.840082:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001cae17e9. 00000800:00000100:2.0:1784949263.988081:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.6@tcp 00000800:00000010:2.0:1784949263.988090:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a21d8ef3. 00000400:00000200:2.0:1784949263.988100:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949263.988111:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949263.988122:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11537 health :900 00000400:00000200:2.0:1784949263.988131:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949263.988137:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:2.0:1784949263.988145:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000800:00000100:2.0:1784949263.988181:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000800:00000010:2.0:1784949263.988186:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cee99234. 00000400:00000200:2.0:1784949263.988195:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949263.988203:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949263.988213:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949263.988213:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949263.988224:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:1.0:1784949263.988230:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 00000400:00000200:2.0:1784949263.988235:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000001:1.0:1784949263.988245:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949263.988250:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949263.988253:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:1.0:1784949263.988256:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000400:00000200:1.0:1784949263.988263:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949263.988273:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:1.0:1784949263.988280:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf0e6e18. 00000400:00000200:1.0:1784949263.988288:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-110 00000400:00000200:1.0:1784949263.988293:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -110 00000400:00000200:1.0:1784949263.988299:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:1.0:1784949263.988305:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:1.0:1784949263.988309:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949263.988311:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949263.988313:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:1.0:1784949263.988315:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000400:00000200:1.0:1784949263.988318:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949263.988321:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:1.0:1784949263.988323:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e6d8b8ad. 00000400:00000200:1.0:1784949263.988327:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:1.0:1784949263.988329:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:1.0:1784949263.988332:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00010000:00000001:2.0:1784949264.628065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949264.628077:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949264.628089:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949264.628092:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949264.628175:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949264.628182:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949264.628186:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1784949265.011991:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.10@tcp 00000400:00000200:3.0:1784949265.011991:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000010:2.0:1784949265.011998:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b9b32818. 00000400:00000010:3.0:1784949265.012000:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000ff2aa94d. 00000400:00000200:2.0:1784949265.012006:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949265.012008:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000072ce5ef0. 00000400:00000200:2.0:1784949265.012013:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:3.0:1784949265.012014:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ba47b3fb. 00000400:00000200:2.0:1784949265.012021:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11538 health :700 00000400:00000200:2.0:1784949265.012026:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:3.0:1784949265.012029:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000ca3adacb. 00000400:00000200:2.0:1784949265.012030:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:2.0:1784949265.012049:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:3.0:1784949265.012050:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949265.012073:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949265.012079:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949265.012083:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3809:3809:246:4294967295] 192.168.202.137@tcp LPNI seq info [2444:2444:8:4294967295] 00000400:00000200:3.0:1784949265.012089:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949265.012094:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949265.012098:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949265.012102:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005f459fde. 00000400:00000200:1.0:1784949265.012104:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784949265.012110:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949265.012113:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:1.0:1784949265.012120:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949265.012123:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:3.0:1784949265.012124:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:1.0:1784949265.012127:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 00000800:00000200:3.0:1784949265.012128:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 2, nob 96 niov 1 nkiov 0 00000400:00000010:1.0:1784949265.012129:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000400:00000200:1.0:1784949265.012136:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949265.012140:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:1.0:1784949265.012143:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000400:00000200:1.0:1784949265.012148:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-110 00000400:00000200:1.0:1784949265.012151:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -110 00000400:00000010:3.0:1784949265.012151:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 000000003e9ea21c. 00000400:00000200:1.0:1784949265.012156:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000400:02000040:3.0:1784949265.012160:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000800:00000200:2.0:1784949265.012262:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949265.012265:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005f459fde. 00000400:00000200:2.0:1784949265.012268:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949265.012272:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949265.012274:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949265.012277:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949265.012284:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949265.012292:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949265.012294:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949265.012298:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949265.012304:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949265.012315:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949265.012324:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949265.012331:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949265.012334:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3810:3810:247:4294967295] 192.168.202.137@tcp LPNI seq info [2445:2445:8:4294967295] 00000400:00000200:2.0:1784949265.012340:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949265.012343:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949265.012347:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000800:00000200:2.0:1784949265.012350:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949265.012354:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949265.012356:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949265.012389:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949265.012420:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949265.012424:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000400:00000200:2.0:1784949265.012429:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949265.012433:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949265.012434:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949265.012437:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949265.012444:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949265.012447:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949265.012451:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949265.012457:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949265.012468:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x943d 00000800:00000001:0.0:1784949265.012471:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949265.012477:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949265.012478:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949265.012481:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949265.012494:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba47b3fb 00000400:00000010:0.0:1784949265.012496:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ba47b3fb. 00000400:00000200:0.0:1784949265.012498:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949265.012500:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000ff2aa94d. 00000400:00000010:0.0:1784949265.012502:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000072ce5ef0. 00000800:00000001:0.0:1784949265.012523:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949265.012524:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949265.013973:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 000000003e9ea21c. 00010000:00000001:2.0:1784949265.651973:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949265.651979:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949265.651985:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949265.651986:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949265.652075:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949265.652079:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949265.652082:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1784949266.036032:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000800:00000010:2.0:1784949266.036040:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c2b2d4f6. 00000400:00000200:2.0:1784949266.036050:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949266.036058:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949266.036065:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11539 health :900 00000400:00000200:2.0:1784949266.036071:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949266.036076:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:2.0:1784949266.036081:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:1.0:1784949266.036144:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949266.036155:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:1.0:1784949266.036163:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949266.036165:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949266.036169:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:1.0:1784949266.036172:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:1.0:1784949266.036177:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949266.036182:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:1.0:1784949266.036187:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005152bcfe. 00000400:00000200:1.0:1784949266.036194:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:1.0:1784949266.036197:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000400:00000200:1.0:1784949266.036201:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00010000:00000001:2.0:1784949266.675984:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949266.675990:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949266.675995:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949266.675996:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949266.676071:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949266.676074:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949266.676077:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:3.0:1784949267.059978:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949267.060005:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000002751ffe5. 00000800:00000100:2.0:1784949267.060020:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000400:00000010:3.0:1784949267.060023:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009f047fe3. 00000800:00000010:2.0:1784949267.060029:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000010:3.0:1784949267.060034:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002eb3bd19. 00000400:00000200:2.0:1784949267.060041:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949267.060048:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008c288862. 00000400:00000200:2.0:1784949267.060055:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949267.060065:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949267.060073:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11540 health :800 00000400:00000200:2.0:1784949267.060087:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949267.060092:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:3.0:1784949267.060099:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949267.060109:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949267.060115:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3811:3811:248:4294967295] 192.168.202.137@tcp LPNI seq info [2446:2446:8:4294967295] 00000400:00000200:3.0:1784949267.060125:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949267.060134:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949267.060142:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949267.060146:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000800:00000010:3.0:1784949267.060152:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949267.060166:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949267.060178:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949267.060184:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949267.060232:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949267.060245:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:1.0:1784949267.060255:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949267.060259:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949267.060262:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:1.0:1784949267.060266:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000400:00000200:1.0:1784949267.060272:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949267.060277:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:1.0:1784949267.060282:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006200396a. 00000400:00000200:1.0:1784949267.060290:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:1.0:1784949267.060294:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:1.0:1784949267.060299:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000800:00000200:2.0:1784949267.060386:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949267.060392:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:2.0:1784949267.060398:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949267.060405:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949267.060410:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:2.0:1784949267.060414:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949267.060427:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949267.060440:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949267.060444:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949267.060466:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949267.060471:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949267.060481:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949267.060489:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949267.060496:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949267.060499:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3812:3812:249:4294967295] 192.168.202.137@tcp LPNI seq info [2447:2447:8:4294967295] 00000400:00000200:2.0:1784949267.060506:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949267.060509:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949267.060512:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c2b2d4f6. 00000800:00000200:2.0:1784949267.060515:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949267.060519:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949267.060521:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c2b2d4f6 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949267.060551:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949267.060581:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949267.060588:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949267.060591:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949267.060591:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949267.060593:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949267.060595:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c2b2d4f6. 00000400:00000200:0.0:1784949267.060602:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949267.060602:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949267.060609:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949267.060618:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9441 00000800:00000001:0.0:1784949267.060624:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949267.060631:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949267.060633:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949267.060636:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949267.060641:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002eb3bd19 00000400:00000010:0.0:1784949267.060643:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002eb3bd19. 00000400:00000200:0.0:1784949267.060646:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949267.060648:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000002751ffe5. 00000400:00000010:0.0:1784949267.060651:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009f047fe3. 00000800:00000001:0.0:1784949267.060656:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949267.060660:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949267.060660:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949267.700006:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949267.700015:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949267.700023:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949267.700025:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949267.700142:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949267.700149:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949267.700153:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1784949268.084006:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.29@tcp 00000800:00000010:2.0:1784949268.084015:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:2.0:1784949268.084027:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949268.084036:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.29@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949268.084045:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11541 health :900 00000400:00000200:2.0:1784949268.084053:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949268.084058:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.29@tcp: 1 00000400:00000200:2.0:1784949268.084065:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.29@tcp: -114 00000800:00000100:2.0:1784949268.084096:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.6@tcp 00000800:00000010:2.0:1784949268.084099:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000200:2.0:1784949268.084104:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949268.084110:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949268.084115:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949268.084120:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:2.0:1784949268.084125:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:1.0:1784949268.084142:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949268.084155:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.29@tcp(0000000043d62367) state 0x840 00000400:00000001:1.0:1784949268.084166:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.084169:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.084173:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000010:1.0:1784949268.084177:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000400:00000200:1.0:1784949268.084183:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.084189:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.29@tcp: -114 00000400:00000010:1.0:1784949268.084196:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c49404b8. 00000400:00000200:1.0:1784949268.084206:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.29@tcp:-110 00000400:00000200:1.0:1784949268.084211:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.29@tcp(0000000043d62367) state 0x2040 rc -110 00000400:00000200:1.0:1784949268.084218:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.29@tcp 00000400:00000200:1.0:1784949268.084245:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000400:00000001:1.0:1784949268.084251:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.084254:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.084257:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:1.0:1784949268.084261:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000400:00000200:1.0:1784949268.084266:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.084271:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:1.0:1784949268.084277:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000063f28b6f. 00000400:00000200:1.0:1784949268.084286:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-110 00000400:00000200:1.0:1784949268.084291:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -110 00000400:00000200:1.0:1784949268.084298:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:3.0:1784949268.084328:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.29@tcp NID 111.222.142.29@tcp: -110. discovery complete 00000400:00000200:3.0:1784949268.084344:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.142.29@tcp primary NID 111.222.142.29@tcp rc -110 00000100:00000010:3.0:1784949268.084369:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000876612f3. 00000100:00000001:3.0:1784949268.084410:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000100:00000040:3.0:1784949268.084416:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000876612f3 refcount 1 to 111.222.142.29@tcp 00010000:00000010:3.0:1784949268.084424:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 000000003e9ea21c. 00010000:00080000:3.0:1784949268.084505:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.142.29@tcp at tail 00010000:00000001:3.0:1784949268.084509:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949268.084513:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949268.084527:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 00000000c38711c7. 00000100:00000400:3.0:1784949268.084532:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.173.30@tcp: rc = -75 00000100:00000040:3.0:1784949268.084535:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.173.30@tcp cnt=32 00000400:00000200:3.0:1784949268.084539:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784949268.084542:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784949268.084546:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784949268.084548:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784949268.084550:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784949268.084557:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784949268.084563:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.084566:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084569:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784949268.084572:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784949268.084576:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949268.084592:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784949268.084595:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084597:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.084600:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084602:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784949268.084605:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784949268.084608:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949268.084613:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784949268.084615:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084616:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.084619:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084621:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784949268.084623:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784949268.084626:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949268.084630:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784949268.084632:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084634:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.084636:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084639:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:3.0:1784949268.084642:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:3.0:1784949268.084646:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949268.084651:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:3.0:1784949268.084654:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084656:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.084659:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084661:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:3.0:1784949268.084663:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:1.0:1784949268.084664:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949268.084669:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.084673:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784949268.084680:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784949268.084683:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084686:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000010:1.0:1784949268.084690:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004f865a97. 00000400:00000200:3.0:1784949268.084690:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084696:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000010:1.0:1784949268.084698:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000400:00000200:3.0:1784949268.084705:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000010:1.0:1784949268.084707:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f12dd04d. 00000400:00000200:3.0:1784949268.084711:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.084717:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949268.084722:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000400:00000200:3.0:1784949268.084725:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084745:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.084748:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084751:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:3.0:1784949268.084753:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:3.0:1784949268.084756:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.084762:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:1.0:1784949268.084767:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.084771:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.084774:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.084776:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:1.0:1784949268.084779:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3813:3813:251:4294967295] 111.222.173.1@tcp LPNI seq info [54:54:8:4294967295] 00000400:00000200:1.0:1784949268.084784:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:1.0:1784949268.084790:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.084794:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000800:00000010:1.0:1784949268.084799:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000600b3b47. 00000400:00000200:3.0:1784949268.084799:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:3.0:1784949268.084806:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.0:1784949268.084810:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949268.084812:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.1:1784949268.084818:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949268.084822:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084827:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784949268.084829:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:3.0:1784949268.084834:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.084838:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:1.0:1784949268.084841:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:3.0:1784949268.084846:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:3.0:1784949268.084849:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084853:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.084855:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784949268.084861:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084866:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000010:1.0:1784949268.084867:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004520da55. 00000400:00000200:3.0:1784949268.084869:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000010:1.0:1784949268.084872:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000400:00000010:1.0:1784949268.084877:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009e0071c7. 00000400:00000200:3.0:1784949268.084877:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.084884:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949268.084892:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:3.0:1784949268.084896:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.084901:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.084932:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.084935:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:1.0:1784949268.084939:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784949268.084939:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:1.0:1784949268.084948:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949268.084950:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.084953:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.084958:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.084960:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:1.0:1784949268.084962:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3814:3814:250:4294967295] 111.222.142.1@tcp LPNI seq info [52:52:8:4294967295] 00000400:00000200:1.0:1784949268.084966:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:1.0:1784949268.084970:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.084973:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000800:00000010:1.0:1784949268.084977:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e706e181. 00000400:00000200:3.0:1784949268.084999:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000800:00000200:1.0:1784949268.085009:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949268.085010:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.0:1784949268.085016:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784949268.085019:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.1:1784949268.085034:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949268.085040:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000800:00000200:1.1:1784949268.085042:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.1 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949268.085056:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949268.085057:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000800:00000010:1.0:1784949268.085060:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e706e181. 00000400:00000200:3.0:1784949268.085070:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000200:1.0:1784949268.085073:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949268.085084:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085085:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949268.085101:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949268.085114:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:3.0:1784949268.085114:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:3.0:1784949268.085126:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949268.085137:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:3.0:1784949268.085144:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.085153:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085158:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784949268.085161:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:3.0:1784949268.085170:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085174:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:1.0:1784949268.085184:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 rc 0 00000400:00000200:3.0:1784949268.085194:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784949268.085200:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.085207:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.085213:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784949268.085222:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085230:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:3.0:1784949268.085234:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000010:1.0:1784949268.085236:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000b241612. 00000400:00000200:3.0:1784949268.085244:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949268.085250:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000010:1.0:1784949268.085262:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000041ec6752. 00000400:00000200:1.0:1784949268.085271:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:1.0:1784949268.085280:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:1.0:1784949268.085285:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.085290:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.085295:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.085299:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:1.0:1784949268.085304:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3815:3815:250:4294967295] 111.222.173.2@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:1.0:1784949268.085312:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:1.0:1784949268.085320:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.085326:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:1.0:1784949268.085330:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e706e181. 00000800:00000200:1.0:1784949268.085336:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000800:00000200:1.1:1784949268.085342:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:1.0:1784949268.085346:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:1.0:1784949268.085349:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:1.0:1784949268.085354:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000010:1.0:1784949268.085359:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c49404b8. 00000400:00000010:1.0:1784949268.085363:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 00000400:00000200:3.0:1784949268.085377:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:3.0:1784949268.085383:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949268.085388:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f65f2fb7. 00000400:00000200:3.0:1784949268.085390:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.085404:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949268.085410:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085419:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:3.0:1784949268.085426:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:1.0:1784949268.085427:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:1.0:1784949268.085439:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949268.085446:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085465:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.085469:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.085472:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:1.0:1784949268.085474:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3816:3816:249:4294967295] 111.222.142.2@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:1.0:1784949268.085477:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:1.0:1784949268.085481:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.085484:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000800:00000010:1.0:1784949268.085487:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000be1f3156. 00000800:00000200:1.0:1784949268.085493:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949268.085493:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784949268.085500:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.1:1784949268.085501:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:1.0:1784949268.085506:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:3.0:1784949268.085508:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.085509:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x261 rc 0 00000400:00000200:3.0:1784949268.085517:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085521:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:1.0:1784949268.085524:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784949268.085529:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000010:1.0:1784949268.085530:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005152bcfe. 00000400:00000010:1.0:1784949268.085535:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000010:1.0:1784949268.085538:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003b463cff. 00000400:00000200:3.0:1784949268.085539:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085545:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784949268.085552:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784949268.085555:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.085559:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.085562:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085565:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:1.0:1784949268.085569:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:3.0:1784949268.085572:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:1.0:1784949268.085574:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.085580:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.085584:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949268.085584:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085588:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:1.0:1784949268.085592:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3817:3817:248:4294967295] 111.222.173.3@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:1.0:1784949268.085595:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:1.0:1784949268.085598:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.085600:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:1.0:1784949268.085603:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fee8262a. 00000400:00000200:3.0:1784949268.085607:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000800:00000200:1.0:1784949268.085608:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949268.085614:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.1:1784949268.085615:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949268.085621:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.085626:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085629:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:3.0:1784949268.085632:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:1.0:1784949268.085633:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:1.0:1784949268.085637:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:1.0:1784949268.085643:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:3.0:1784949268.085643:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949268.085649:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000010:1.0:1784949268.085653:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000400:00000200:3.0:1784949268.085657:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000200:3.0:1784949268.085661:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949268.085664:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000091a157d3. 00000400:00000200:3.0:1784949268.085665:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.085673:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784949268.085677:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085681:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784949268.085683:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:1.0:1784949268.085686:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:1.0:1784949268.085691:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949268.085692:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085696:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.085699:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.085701:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:1.0:1784949268.085703:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3818:3818:247:4294967295] 111.222.142.3@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:1.0:1784949268.085706:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:1.0:1784949268.085710:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.085714:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:1.0:1784949268.085718:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005c010cf6. 00000400:00000200:3.0:1784949268.085722:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000800:00000200:1.0:1784949268.085724:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949268.085728:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.0:1784949268.085765:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949268.085776:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949268.085779:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.1:1784949268.085781:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.3 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949268.085794:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.3@tcp 00000800:00000010:1.0:1784949268.085815:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005c010cf6. 00000400:00000200:3.0:1784949268.085815:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:1.0:1784949268.085819:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949268.085821:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:1.0:1784949268.085827:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949268.085830:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:1.0:1784949268.085831:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949268.085837:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:3.0:1784949268.085841:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085842:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000200:1.0:1784949268.085850:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:1.0:1784949268.085853:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 rc 0 00000400:00000200:1.0:1784949268.085856:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000010:1.0:1784949268.085859:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e6d8b8ad. 00000400:00000010:1.0:1784949268.085862:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000400:00000010:1.0:1784949268.085863:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006d2c4b24. 00000400:00000200:1.0:1784949268.085866:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:1.0:1784949268.085869:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:1.0:1784949268.085871:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.085873:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.085875:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.085877:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:1.0:1784949268.085879:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3819:3819:247:4294967295] 111.222.173.4@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:1.0:1784949268.085882:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:1.0:1784949268.085885:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.085889:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000800:00000010:1.0:1784949268.085891:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000005c010cf6. 00000800:00000200:1.0:1784949268.085894:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000800:00000200:1.1:1784949268.085897:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:1.0:1784949268.085899:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:1.0:1784949268.085902:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784949268.085938:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:3.0:1784949268.085942:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.085946:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.085949:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000200:3.0:1784949268.085954:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.085958:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000010:1.0:1784949268.085961:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000bf0e6e18. 00000400:00000200:3.0:1784949268.085962:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000010:1.0:1784949268.085966:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007d821c01. 00000400:00000010:1.0:1784949268.085970:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000171e706a. 00000400:00000200:3.0:1784949268.085971:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.085978:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:3.0:1784949268.085984:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:3.0:1784949268.085987:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.085990:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.085993:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086012:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:1.0:1784949268.086021:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:3.0:1784949268.086023:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:1.0:1784949268.086030:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.086038:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949268.086044:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.086048:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.086057:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:1.0:1784949268.086060:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3820:3820:246:4294967295] 111.222.173.5@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:1.0:1784949268.086064:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:1.0:1784949268.086070:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.086076:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000800:00000010:1.0:1784949268.086080:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004b264730. 00000400:00000200:3.0:1784949268.086087:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000800:00000200:1.0:1784949268.086091:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:3.0:1784949268.086096:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.1:1784949268.086105:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:3.0:1784949268.086107:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.086118:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086125:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000200:1.0:1784949268.086130:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:3.0:1784949268.086138:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:1.0:1784949268.086140:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x261 rc 0 00000400:00000200:1.0:1784949268.086155:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000200:3.0:1784949268.086157:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949268.086162:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006200396a. 00000400:00000010:1.0:1784949268.086170:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000200:3.0:1784949268.086177:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784949268.086183:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949268.086189:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000339be500. 00000400:00000200:3.0:1784949268.086190:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.086204:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:1.0:1784949268.086212:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:3.0:1784949268.086216:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:3.0:1784949268.086226:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:1.0:1784949268.086232:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:3.0:1784949268.086243:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.086245:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.086258:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.086263:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.086269:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:1.0:1784949268.086276:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3821:3821:245:4294967295] 111.222.142.5@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:1.0:1784949268.086286:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:1.0:1784949268.086298:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.086309:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:1.0:1784949268.086317:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008d5370ec. 00000400:00000200:3.0:1784949268.086324:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784949268.086335:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.0:1784949268.086336:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:3.0:1784949268.086345:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949268.086348:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784949268.086362:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000800:00000200:1.1:1784949268.086371:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:3.0:1784949268.086374:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000800:00000200:1.1:1784949268.086380:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.5 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784949268.086389:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000800:00000100:1.0:1784949268.086396:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.5@tcp 00000800:00000010:1.0:1784949268.086401:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008d5370ec. 00000400:00000200:1.0:1784949268.086406:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949268.086408:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.086418:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949268.086430:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:3.0:1784949268.086436:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949268.086464:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949268.086464:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.086470:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:3.0:1784949268.086474:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086478:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:1.0:1784949268.086482:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:3.0:1784949268.086483:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:3.0:1784949268.086491:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.086492:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:1.0:1784949268.086495:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 rc 0 00000400:00000200:1.0:1784949268.086498:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:1.0:1784949268.086501:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000d531934e. 00000400:00000010:1.0:1784949268.086504:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000010:1.0:1784949268.086506:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a8bf092e. 00000400:00000200:1.0:1784949268.086508:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:1.0:1784949268.086511:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:1.0:1784949268.086513:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.086515:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.086517:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.086519:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:1.0:1784949268.086521:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3822:3822:245:4294967295] 111.222.173.6@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:1.0:1784949268.086523:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:1.0:1784949268.086527:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.086530:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:1.0:1784949268.086532:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008d5370ec. 00000800:00000200:1.0:1784949268.086534:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784949268.086538:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:1.0:1784949268.086541:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:1.0:1784949268.086545:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:3.0:1784949268.086552:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:3.0:1784949268.086555:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.086558:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.086560:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:1.0:1784949268.086565:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:3.0:1784949268.086567:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000001:1.0:1784949268.086568:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.086570:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:1.0:1784949268.086572:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000400:00000200:3.0:1784949268.086575:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000200:1.0:1784949268.086576:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.086581:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000200:3.0:1784949268.086583:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000010:1.0:1784949268.086586:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004520da55. 00000400:00000200:3.0:1784949268.086591:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.086592:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-5 00000400:00000200:1.0:1784949268.086598:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -5 00000400:00000200:3.0:1784949268.086604:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784949268.086607:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949268.086611:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:3.0:1784949268.086614:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.086620:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086623:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:1.0:1784949268.086626:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000200:3.0:1784949268.086629:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000010:1.0:1784949268.086631:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004520da55. 00000400:00000010:1.0:1784949268.086635:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000400:00000200:3.0:1784949268.086637:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949268.086639:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000cef8b3cd. 00000400:00000200:1.0:1784949268.086645:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:3.0:1784949268.086649:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:3.0:1784949268.086652:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949268.086655:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.086658:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086661:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000200:3.0:1784949268.086664:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:1.0:1784949268.086665:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:1.0:1784949268.086672:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949268.086675:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.086677:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.086681:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.086683:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:1.0:1784949268.086684:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3823:3823:244:4294967295] 111.222.142.6@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:1.0:1784949268.086688:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:1.0:1784949268.086691:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.086694:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:1.0:1784949268.086696:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b80ce7eb. 00000400:00000200:3.0:1784949268.086702:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000800:00000200:1.0:1784949268.086703:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000200:3.0:1784949268.086709:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.1:1784949268.086711:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000400:00000200:3.0:1784949268.086715:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.086716:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:1.0:1784949268.086720:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x261 rc 0 00000400:00000200:3.0:1784949268.086725:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086748:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:1.0:1784949268.086752:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000200:3.0:1784949268.086753:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:3.0:1784949268.086760:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949268.086761:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:1.0:1784949268.086765:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000400:00000200:3.0:1784949268.086768:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784949268.086772:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949268.086776:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009ec02d9a. 00000400:00000200:3.0:1784949268.086776:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949268.086784:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:3.0:1784949268.086787:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086791:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:3.0:1784949268.086794:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000200:1.0:1784949268.086796:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:1.0:1784949268.086801:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949268.086803:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949268.086806:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.086810:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.086812:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:1.0:1784949268.086814:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3824:3824:243:4294967295] 111.222.173.7@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:1.0:1784949268.086817:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:1.0:1784949268.086820:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.086823:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:1.0:1784949268.086825:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e4c24e27. 00000400:00000200:3.0:1784949268.086829:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000800:00000200:1.0:1784949268.086831:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:3.0:1784949268.086834:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:1.1:1784949268.086837:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:3.0:1784949268.086840:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949268.086847:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949268.086851:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000400:00000200:1.0:1784949268.086854:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:1.0:1784949268.086858:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000100:00000010:3.0:1784949268.086858:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 00000000c38711c7. 00000400:00000200:1.0:1784949268.086865:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:1.0:1784949268.086869:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eb983bef. 00000020:00000001:3.0:1784949268.086873:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000010:1.0:1784949268.086874:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b039801a. 00000400:00000010:1.0:1784949268.086879:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e3882e96. 00010000:00000001:3.0:1784949268.086880:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000400:00000200:1.0:1784949268.086884:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000100:00000001:3.0:1784949268.086887:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:1.0:1784949268.086891:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:1.0:1784949268.086894:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.086896:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.086897:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.086900:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:1.0:1784949268.086921:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3825:3825:242:4294967295] 111.222.142.7@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:1.0:1784949268.086925:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:1.0:1784949268.086931:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.086934:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:1.0:1784949268.086938:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000bc928ba2. 00000400:00000010:3.0:1784949268.086940:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000bf7eadde. 00000800:00000200:1.0:1784949268.086943:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.0:1784949268.086947:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784949268.086951:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000bf7eadde nid 111.222.173.30@tcp 00000800:00000200:1.1:1784949268.086955:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784949268.086958:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.7 (cnted 0, interval 60, 14 secs later) 00000400:00000010:3.0:1784949268.086960:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000000486261c. 00000800:00000100:1.0:1784949268.086963:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.7@tcp 00000800:00000010:1.0:1784949268.086967:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000bc928ba2. 00000400:00000200:3.0:1784949268.086968:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 000000000486261c net tcp 00000400:00000200:1.0:1784949268.086988:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000010:3.0:1784949268.086994:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 00000000dd0652fb. 00000400:00000200:1.0:1784949268.086999:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949268.087008:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949268.087014:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 00000000dd0652fb nid 111.222.173.30@tcp 00000400:00000200:1.0:1784949268.087018:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000200:3.0:1784949268.087037:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.173.30@tcp NID 111.222.173.30@tcp flags 0x0 00000400:00000200:3.0:1784949268.087049:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.173.30@tcp: 0 00000400:00000200:1.0:1784949268.087051:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000200:1.0:1784949268.087068:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:1.0:1784949268.087073:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 rc 0 00000400:00000200:3.0:1784949268.087080:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.30@tcp: 0 00000400:00000200:3.0:1784949268.087087:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949268.087096:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:1.0:1784949268.087102:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000e120de1c. 00000400:00000010:1.0:1784949268.087105:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000010:1.0:1784949268.087109:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000014336a14. 00000400:00000200:1.0:1784949268.087116:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:1.0:1784949268.087127:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:1.0:1784949268.087133:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087137:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087140:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087144:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:1.0:1784949268.087149:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3826:3826:242:4294967295] 111.222.173.8@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:1.0:1784949268.087155:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:1.0:1784949268.087165:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087170:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:1.0:1784949268.087173:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000bc928ba2. 00000800:00000200:1.0:1784949268.087178:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784949268.087182:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:1.0:1784949268.087207:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:1.0:1784949268.087210:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087214:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:1.0:1784949268.087218:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005cc3cd5d. 00000400:00000010:1.0:1784949268.087221:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000400:00000010:1.0:1784949268.087225:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a3f431e3. 00000400:00000200:1.0:1784949268.087229:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:1.0:1784949268.087235:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:1.0:1784949268.087239:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087243:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087247:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087252:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:1.0:1784949268.087258:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3827:3827:241:4294967295] 111.222.173.9@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:1.0:1784949268.087264:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:1.0:1784949268.087272:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087279:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:1.0:1784949268.087283:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c158c1d0. 00000800:00000200:1.0:1784949268.087290:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784949268.087296:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000400:00000200:1.0:1784949268.087318:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:1.0:1784949268.087322:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087326:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:1.0:1784949268.087331:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001a4eba99. 00000400:00000010:1.0:1784949268.087335:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000400:00000010:1.0:1784949268.087340:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000007e2f3d0. 00000400:00000200:1.0:1784949268.087343:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:1.0:1784949268.087347:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:1.0:1784949268.087351:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087354:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087356:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087359:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:1.0:1784949268.087362:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3828:3828:240:4294967295] 111.222.142.9@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:1.0:1784949268.087367:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:1.0:1784949268.087373:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087377:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:1.0:1784949268.087380:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002ddfcc2d. 00000800:00000200:1.0:1784949268.087384:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.0:1784949268.087388:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949268.087393:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784949268.087396:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.9 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949268.087404:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.9@tcp 00000800:00000010:1.0:1784949268.087406:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002ddfcc2d. 00000400:00000200:1.0:1784949268.087409:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949268.087413:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949268.087418:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949268.087423:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:1.0:1784949268.087428:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000200:1.0:1784949268.087433:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:1.0:1784949268.087435:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 rc 0 00000400:00000200:1.0:1784949268.087439:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 00000400:00000001:1.0:1784949268.087443:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.087446:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.087448:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:1.0:1784949268.087467:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000400:00000200:1.0:1784949268.087469:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.087473:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:1.0:1784949268.087474:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000400:00000200:1.0:1784949268.087479:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-5 00000400:00000200:1.0:1784949268.087480:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -5 00000400:00000200:1.0:1784949268.087483:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00000400:00000200:1.0:1784949268.087485:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:1.0:1784949268.087489:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000010:1.0:1784949268.087491:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000400:00000010:1.0:1784949268.087493:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f9fb66e4. 00000400:00000200:1.0:1784949268.087495:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:1.0:1784949268.087498:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:1.0:1784949268.087500:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087501:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087504:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087506:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:1.0:1784949268.087509:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3829:3829:240:4294967295] 111.222.142.10@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:1.0:1784949268.087512:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:1.0:1784949268.087518:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087522:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:1.0:1784949268.087523:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002ddfcc2d. 00000800:00000200:1.0:1784949268.087525:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784949268.087528:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000400:00000200:1.0:1784949268.087530:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:1.0:1784949268.087531:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087533:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:1.0:1784949268.087535:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef8e57c8. 00000400:00000010:1.0:1784949268.087536:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000400:00000010:1.0:1784949268.087538:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d3823a3e. 00000400:00000200:1.0:1784949268.087540:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:1.0:1784949268.087543:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:1.0:1784949268.087544:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087546:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087548:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087549:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:1.0:1784949268.087551:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3830:3830:239:4294967295] 111.222.173.11@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:1.0:1784949268.087553:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:1.0:1784949268.087557:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087560:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:1.0:1784949268.087562:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a1bdc144. 00000800:00000200:1.0:1784949268.087564:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:1.1:1784949268.087567:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:1.0:1784949268.087576:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:1.0:1784949268.087578:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087580:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:1.0:1784949268.087583:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006b8bfa56. 00000400:00000010:1.0:1784949268.087585:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000400:00000010:1.0:1784949268.087587:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000e3180272. 00000400:00000200:1.0:1784949268.087589:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:1.0:1784949268.087591:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:1.0:1784949268.087593:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087595:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087596:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087598:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:1.0:1784949268.087600:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3831:3831:238:4294967295] 111.222.142.11@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:1.0:1784949268.087603:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:1.0:1784949268.087607:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087611:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:1.0:1784949268.087613:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009dcd7bb6. 00000800:00000200:1.0:1784949268.087616:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.0:1784949268.087617:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949268.087620:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784949268.087622:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.11 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949268.087626:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.11@tcp 00000800:00000010:1.0:1784949268.087628:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009dcd7bb6. 00000400:00000200:1.0:1784949268.087629:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949268.087631:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949268.087633:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949268.087634:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:1.0:1784949268.087636:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:1.0:1784949268.087639:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:1.0:1784949268.087642:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 rc 0 00000400:00000200:1.0:1784949268.087646:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:1.0:1784949268.087650:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000075318cf7. 00000400:00000010:1.0:1784949268.087653:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000400:00000010:1.0:1784949268.087656:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009f8441cb. 00000400:00000200:1.0:1784949268.087658:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:1.0:1784949268.087662:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:1.0:1784949268.087664:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087665:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087667:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087668:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:1.0:1784949268.087670:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3832:3832:238:4294967295] 111.222.173.12@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:1.0:1784949268.087672:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:1.0:1784949268.087677:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087679:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:1.0:1784949268.087682:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009dcd7bb6. 00000800:00000200:1.0:1784949268.087684:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784949268.087687:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:1.0:1784949268.087688:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:1.0:1784949268.087689:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087691:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:1.0:1784949268.087696:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000047e19e91. 00000400:00000010:1.0:1784949268.087698:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000010:1.0:1784949268.087700:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000851e368. 00000400:00000200:1.0:1784949268.087703:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:1.0:1784949268.087706:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:1.0:1784949268.087709:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087711:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087713:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087715:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:1.0:1784949268.087717:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3833:3833:237:4294967295] 111.222.142.12@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:1.0:1784949268.087720:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:1.0:1784949268.087723:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087725:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:1.0:1784949268.087727:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000af953512. 00000800:00000200:1.0:1784949268.087743:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784949268.087746:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:1.0:1784949268.087748:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:1.0:1784949268.087749:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087751:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x861 00000400:00000001:1.0:1784949268.087753:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.087754:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.087755:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:1.0:1784949268.087756:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000400:00000200:1.0:1784949268.087758:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.087760:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:1.0:1784949268.087761:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006200396a. 00000400:00000200:1.0:1784949268.087765:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-5 00000400:00000200:1.0:1784949268.087767:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -5 00000400:00000200:1.0:1784949268.087769:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000400:00000200:1.0:1784949268.087771:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:1.0:1784949268.087774:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002c1a3d41. 00000400:00000010:1.0:1784949268.087775:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000010:1.0:1784949268.087777:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000022dbcdce. 00000400:00000200:1.0:1784949268.087779:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:1.0:1784949268.087782:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:1.0:1784949268.087784:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087786:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087788:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087790:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:1.0:1784949268.087792:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3834:3834:236:4294967295] 111.222.173.13@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:1.0:1784949268.087796:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:1.0:1784949268.087799:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087802:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:1.0:1784949268.087804:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000800:00000200:1.0:1784949268.087807:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784949268.087809:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:1.0:1784949268.087818:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:1.0:1784949268.087820:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087822:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:1.0:1784949268.087824:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006200396a. 00000400:00000010:1.0:1784949268.087826:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000010:1.0:1784949268.087829:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003859b1c5. 00000400:00000200:1.0:1784949268.087831:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:1.0:1784949268.087833:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:1.0:1784949268.087835:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087837:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087838:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087840:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:1.0:1784949268.087842:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3835:3835:235:4294967295] 111.222.142.13@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:1.0:1784949268.087844:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:1.0:1784949268.087847:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087850:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:1.0:1784949268.087851:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000800:00000200:1.0:1784949268.087854:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.0:1784949268.087855:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.13 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949268.087857:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784949268.087859:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.13 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949268.087861:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.13@tcp 00000800:00000010:1.0:1784949268.087862:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000200:1.0:1784949268.087864:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949268.087867:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949268.087869:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949268.087871:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:1.0:1784949268.087874:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:1.0:1784949268.087877:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:1.0:1784949268.087879:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 rc 0 00000400:00000200:1.0:1784949268.087883:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:1.0:1784949268.087886:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001c9428b0. 00000400:00000010:1.0:1784949268.087888:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000010:1.0:1784949268.087893:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009234a5fd. 00000400:00000200:1.0:1784949268.087896:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:1.0:1784949268.087899:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:1.0:1784949268.087902:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.087924:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.087927:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.087929:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:1.0:1784949268.087931:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3836:3836:235:4294967295] 111.222.142.14@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:1.0:1784949268.087935:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:1.0:1784949268.087941:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.087945:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:1.0:1784949268.087948:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000800:00000200:1.0:1784949268.087952:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784949268.087956:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000400:00000200:1.0:1784949268.087959:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:1.0:1784949268.087961:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x261 rc 0 00000400:00000200:1.0:1784949268.087963:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:1.0:1784949268.087967:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000467d4d37. 00000400:00000010:1.0:1784949268.087969:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000400:00000010:1.0:1784949268.087990:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000451e3fe4. 00000400:00000200:1.0:1784949268.087995:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:1.0:1784949268.088001:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:1.0:1784949268.088004:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.088007:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.088010:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.088012:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:1.0:1784949268.088016:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3837:3837:234:4294967295] 111.222.173.15@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:1.0:1784949268.088020:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:1.0:1784949268.088026:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.088030:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:1.0:1784949268.088033:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b9b32818. 00000800:00000200:1.0:1784949268.088037:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784949268.088041:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:1.0:1784949268.088057:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:1.0:1784949268.088060:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:1.0:1784949268.088064:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:1.0:1784949268.088068:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:1.0:1784949268.088071:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000400:00000010:1.0:1784949268.088075:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000acbf6818. 00000400:00000200:1.0:1784949268.088080:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:1.0:1784949268.088085:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:1.0:1784949268.088090:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949268.088093:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.088096:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949268.088099:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:1.0:1784949268.088102:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3838:3838:233:4294967295] 111.222.142.15@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:1.0:1784949268.088107:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:1.0:1784949268.088112:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.088116:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:1.0:1784949268.088120:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cee99234. 00000800:00000200:1.0:1784949268.088124:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.0:1784949268.088126:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.15 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1784949268.088130:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784949268.088133:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.15 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1784949268.088137:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.15@tcp 00000800:00000010:1.0:1784949268.088138:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cee99234. 00000400:00000200:1.0:1784949268.088142:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949268.088145:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949268.088148:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949268.088150:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:1.0:1784949268.088154:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:1.0:1784949268.088159:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:1.0:1784949268.088161:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 rc 0 00000400:00000200:1.0:1784949268.088165:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000400:00000001:1.0:1784949268.088168:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.088170:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.088172:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b039801a 00000400:00000010:1.0:1784949268.088174:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b039801a. 00000400:00000200:1.0:1784949268.088178:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.088185:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:1.0:1784949268.088189:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eb983bef. 00000400:00000200:1.0:1784949268.088193:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-5 00000400:00000200:1.0:1784949268.088195:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -5 00000400:00000200:1.0:1784949268.088200:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:1.0:1784949268.088204:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.30@tcp(00000000bf7eadde) state 0x40 00000400:00000010:1.0:1784949268.088211:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000eb983bef. 00000400:00000010:1.0:1784949268.088216:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b039801a. 00000400:00000010:1.0:1784949268.088224:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008c245eeb. 00000400:00000200:1.0:1784949268.088231:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.30@tcp 00000400:00000200:1.0:1784949268.088240:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.173.30@tcp local destination 00000400:00000200:1.0:1784949268.088247:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949268.088255:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.173.30@tcp 00000400:00000200:1.0:1784949268.088262:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.173.30@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:1.0:1784949268.088269:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3839:3839:233:4294967295] 111.222.173.30@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:1.0:1784949268.088279:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.30@tcp 00000400:00000200:1.0:1784949268.088293:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.30@tcp(111.222.173.30@tcp:111.222.173.30@tcp) : GET try# 0 00000800:00000200:1.0:1784949268.088302:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.30@tcp 00000800:00000010:1.0:1784949268.088308:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cee99234. 00000800:00000010:1.0:1784949268.088319:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 0000000096ccf04d. 00000800:00000010:1.0:1784949268.088325:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 0000000034386f9e. 00000800:00000200:1.0:1784949268.088330:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000096ccf04d] -> 12345-111.222.173.30@tcp (2) 00000800:00000200:1.1:1784949268.088334:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000096ccf04d] -> 12345-111.222.173.30@tcp (2) 00000400:00000200:1.0:1784949268.088354:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.30@tcp 00000400:00000200:1.0:1784949268.088356:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.30@tcp(00000000bf7eadde) state 0x240 rc 0 00000400:00000200:1.0:1784949268.088361:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:1.0:1784949268.088364:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.088366:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.088368:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784949268.088371:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000400:00000200:1.0:1784949268.088376:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.088380:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:1.0:1784949268.088384:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001a4eba99. 00000400:00000200:1.0:1784949268.088391:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-5 00000400:00000200:1.0:1784949268.088394:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -5 00000400:00000200:1.0:1784949268.088397:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:1.0:1784949268.088401:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:1.0:1784949268.088403:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.088404:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.088406:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000400:00000010:1.0:1784949268.088407:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000400:00000200:1.0:1784949268.088409:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.088412:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:1.0:1784949268.088415:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006b8bfa56. 00000400:00000200:1.0:1784949268.088424:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-5 00000400:00000200:1.0:1784949268.088428:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -5 00000400:00000200:1.0:1784949268.088431:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000400:00000200:1.0:1784949268.088440:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000001:1.0:1784949268.088443:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.088444:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.088469:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784949268.088470:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000400:00000200:1.0:1784949268.088472:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.088474:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:1.0:1784949268.088476:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006200396a. 00000400:00000200:1.0:1784949268.088479:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-5 00000400:00000200:1.0:1784949268.088482:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -5 00000400:00000200:1.0:1784949268.088484:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000400:00000200:1.0:1784949268.088488:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:1.0:1784949268.088490:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949268.088491:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949268.088492:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000010:1.0:1784949268.088493:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000400:00000200:1.0:1784949268.088494:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949268.088496:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:1.0:1784949268.088498:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000400:00000200:1.0:1784949268.088501:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-5 00000400:00000200:1.0:1784949268.088502:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -5 00000400:00000200:1.0:1784949268.088505:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784949268.724008:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949268.724017:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949268.724026:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949268.724027:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949268.724139:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949268.724147:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949268.724153:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949268.916030:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1784949268.916037:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949268.916047:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949268.916048:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949268.916056:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1784949268.916060:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046cb422a. 00000100:00000001:2.0:1784949268.916065:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1784949268.916070:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949268.916073:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949268.916079:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949268.916082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949268.916085:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949268.916095:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949268.916095:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949268.916112:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949268.916122:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949268.916134:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949268.916148:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949268.916152:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949268.916161:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046cb422a] to pc [ptlrpcd_00_03+3] req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949268.916208:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949268.916213:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949268.916219:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949268.916224:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949268.916227:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949268.916230:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00100000:0.0:1784949268.916237:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000010:3.0:1784949268.916239:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000100:00000001:0.0:1784949268.916244:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784949268.916247:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949268.916253:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949268.916254:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784949268.916256:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949268.916257:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949268.916260:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949268.916262:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.916267:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949268.916270:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784949268.916273:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:0.0:1784949268.916278:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949268.916281:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949268.916283:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949268.916292:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1784949268.916297:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949268.916344:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949268.916347:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949268.916349:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949268.916352:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949268.916359:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1784949268.916361:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006d868603] to pc [ptlrpcd_00_00+0] req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949268.916365:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949268.916376:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949268.916386:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289685888:0@lo:400: 00000100:00000001:0.0:1784949268.916394:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949268.916401:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949268.916404:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949268.916407:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949268.916412:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949268.916414:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 02000000:00000001:0.0:1784949268.916421:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949268.916423:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11545) 00000400:00000010:0.0:1784949268.916429:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001c023af. 00000400:00000010:0.0:1784949268.916466:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000516cc351. 00000100:00000001:2.0:1784949268.916472:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1784949268.916474:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289685888, portal 25 00000100:00000001:2.0:1784949268.916477:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949268.916478:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949268.916479:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949268.916482:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949268.916483:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554953472 : -115544154598144 : ffff96e9c7b59500) 00000100:00000001:3.0:1784949268.916484:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949268.916494:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949268.916494:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949268.916516:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949268.916522:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949268.916523:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:0.0:1784949268.916526:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000100:00000001:2.0:1784949268.916527:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949268.916530:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289685888, offset 0 02000000:00000001:2.0:1784949268.916534:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949268.916538:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949268.916541:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784949268.916545:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949268.916552:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289686016:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949268.916556:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:0.0:1784949268.916560:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:2.0:1784949268.916561:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949268.916565:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949268.916567:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949268.916572:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000032fe57fd. 02000000:00000001:2.0:1784949268.916574:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949268.916576:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643df80 00000400:00000010:2.0:1784949268.916580:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000ef44eaa. 00000400:00000200:0.0:1784949268.916585:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8aa5 [2] + 0 00000400:00000010:0.0:1784949268.916589:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000014bfbbb1. 00000400:00000200:0.0:1784949268.916592:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000001ee8a23e 00000400:00000010:2.0:1784949268.916597:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:0.0:1784949268.916599:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784949268.916603:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686016, portal 10 00000400:00000200:0.0:1784949268.916605:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949268.916608:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949268.916611:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554951680 : -115544154599936 : ffff96e9c7b58e00) 00000400:00000200:0.0:1784949268.916612:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001ee8a23e 00000400:00000010:0.0:1784949268.916617:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000001ee8a23e. 00000100:00000040:2.0:1784949268.916617:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949268.916626:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784949268.916626:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:0.0:1784949268.916632:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949268.916632:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949268.916635:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000100:00000040:0.0:1784949268.916639:0:337121:0:(events.c:337:request_in_callback()) incoming req@0000000046e93177 x1871651289685888 msgsize 224 00000100:00000200:2.0:1784949268.916639:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289686016, offset 0 00000400:00000200:2.0:1784949268.916655:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784949268.916659:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949268.916662:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949268.916666:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949268.916672:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949268.916672:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949268.916678:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949268.916682:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949268.916685:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949268.916689:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3840:3840:232:4294967295] 192.168.202.137@tcp LPNI seq info [2448:2448:8:4294967295] 00000400:00000200:2.0:1784949268.916701:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949268.916708:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949268.916712:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00000800:00000010:2.0:1784949268.916713:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000031deb17e. 00000400:00000010:0.0:1784949268.916717:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000100:00000001:0.0:1784949268.916756:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949268.916765:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949268.916768:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949268.916771:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949268.916780:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289685888 02000000:00000001:1.0:1784949268.916804:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949268.916809:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:2.0:1784949268.916809:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949268.916810:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949268.916815:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949268.916816:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949268.916819:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000031deb17e type 1, nob 320 niov 1 nkiov 1 02000000:00000001:1.0:1784949268.916821:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949268.916829:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289685888 00000100:00000001:0.0:1784949268.916834:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949268.916835:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949268.916838:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1784949268.916839:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949268.916843:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949268.916845:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949268.916846:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949268.916847:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949268.916854:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949268.916856:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306208298 00000020:00000001:1.0:1784949268.916857:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:2.0:1784949268.916857:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2238 usecs (3 jiffies) 00000100:00000010:0.0:1784949268.916864:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:2.0:1784949268.916866:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306208259 00000100:00000001:1.0:1784949268.916867:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949268.916872:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:0.0:1784949268.916873:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949268.916878:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949268.916882:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:0.0:1784949268.916883:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306208298 00000100:00000001:2.0:1784949268.916884:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306208298 00000020:00000010:1.0:1784949268.916889:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000100:00000001:2.0:1784949268.916890:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306208259 00000100:00000010:0.0:1784949268.916891:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000020:00000010:1.0:1784949268.916897:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000319b9e18. 00000100:00000001:0.0:1784949268.916898:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:1.0:1784949268.916905:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949268.916906:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306208298 00000100:00000010:0.0:1784949268.916913:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000100:00000001:0.0:1784949268.916918:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000040:1.0:1784949268.916918:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949268.916926:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306208298 00000100:00000001:2.0:1784949268.916930:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306208260 00000100:00000001:1.0:1784949268.916944:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949268.916946:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306208299 00000100:00000001:1.0:1784949268.916947:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949268.916947:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306208298 00000100:00000001:1.0:1784949268.916952:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.916956:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306208260 00000100:00000010:0.0:1784949268.916957:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000100:00000001:1.0:1784949268.916961:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949268.916963:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949268.916968:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306208299 00000100:00000001:0.0:1784949268.916969:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949268.916972:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.916974:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949268.916975:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949268.916982:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.916982:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.916986:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949268.917004:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949268.917007:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949268.917007:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949268.917014:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:3.0:1784949268.917019:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949268.917020:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1784949268.917029:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 815 00000100:00000001:2.0:1784949268.917029:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000001:0.0:1784949268.917030:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949268.917031:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000031deb17e. 00000100:00000040:1.0:1784949268.917042:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949268.917043:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949268.917046:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949268.917048:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949268.917051:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818138192 : -115538891413424 : ffff96eb016b6850) 00000100:00000001:2.0:1784949268.917051:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949268.917059:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949268.917060:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949268.917064:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:1.0:1784949268.917067:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000046e93177 x1871651289685888/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:174/0 lens 224/0 e 0 to 0 dl 1784949279 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.917070:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949268.917072:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949268.917078:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000010:3.0:1784949268.917086:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000100:00000001:3.0:1784949268.917098:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784949268.917100:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949268.917102:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:3.0:1784949268.917110:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949268.917113:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:1.0:1784949268.917123:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949268.917127:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e000 00000020:00000010:1.0:1784949268.917133:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:3.0:1784949268.917148:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949268.917149:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 178224 00000100:00100000:1.0:1784949268.917153:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000046e93177 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289685888:12345-0@lo:400: 00000100:00000040:3.0:1784949268.917160:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949268.917166:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:1.0:1784949268.917169:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289685888 00000020:00000001:1.0:1784949268.917178:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:0.0:1784949268.917180:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949268.917184:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949268.917187:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784949268.917188:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949268.917190:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784949268.917194:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949268.917203:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:0.0:1784949268.917204:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784949268.917212:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784949268.917218:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:1.0:1784949268.917222:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000200:0.0:1784949268.917223:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:1.0:1784949268.917231:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1784949268.917234:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f230fea2. 00000100:00000040:0.0:1784949268.917244:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000f230fea2 x1871651289686016 msgsize 224 00000020:00000001:1.0:1784949268.917244:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949268.917253:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00100000:0.0:1784949268.917260:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:1.0:1784949268.917264:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949268.917272:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949268.917277:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949268.917278:0:337113:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:1.0:1784949268.917283:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:0.0:1784949268.917290:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000010:1.0:1784949268.917295:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000017ac11c7. 00000800:00000001:0.0:1784949268.917298:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949268.917301:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949268.917307:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949268.917311:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.917314:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000040:1.0:1784949268.917316:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289685888 00000100:00100000:2.0:1784949268.917321:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686016 00010000:00000001:1.0:1784949268.917323:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:2.0:1784949268.917331:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949268.917335:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00010000:00000200:1.0:1784949268.917337:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000046e93177 x1871651289685888/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:174/0 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.917339:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949268.917348:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949268.917363:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949268.917366:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949268.917370:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=140 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949268.917377:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949268.917381:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949268.917384:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784949268.917388:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949268.917391:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949268.917394:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949268.917398:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949268.917403:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000100:00100000:2.0:1784949268.917407:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686016 00000100:00000200:1.0:1784949268.917408:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289685888, offset 224 00000020:00000001:2.0:1784949268.917414:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949268.917419:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:1.0:1784949268.917425:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:2.0:1784949268.917427:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949268.917433:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949268.917453:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:1.0:1784949268.917458:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:2.0:1784949268.917459:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949268.917465:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949268.917469:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000400:00000200:1.0:1784949268.917470:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643df80 00000100:00000001:2.0:1784949268.917474:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949268.917477:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000400:00000200:1.0:1784949268.917480:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x94b1 [1] + 224 00000020:00000010:2.0:1784949268.917486:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000059e5b358. 00000400:00000200:1.0:1784949268.917487:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949268.917495:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:2.0:1784949268.917495:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000015bbcec8. 00000100:00000001:1.0:1784949268.917499:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:2.0:1784949268.917502:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000200:1.0:1784949268.917506:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949268.917514:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:1.0:1784949268.917517:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.917518:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949268.917522:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949268.917527:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.917533:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949268.917535:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949268.917536:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949268.917540:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949268.917546:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949268.917548:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000010:1.0:1784949268.917551:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000100:00000001:1.0:1784949268.917555:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949268.917557:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949268.917561:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949268.917562:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.917564:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949268.917565:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949268.917567:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949268.917568:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784949268.917570:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949268.917571:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1784949268.917575:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 791 00010000:00000001:1.0:1784949268.917576:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000001:3.0:1784949268.917580:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1784949268.917581:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949268.917581:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949268.917582:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784949268.917585:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784949268.917587:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469555584 : -115544239996032 : ffff96e9c29e8380) 00000020:00000001:1.0:1784949268.917588:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949268.917589:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949268.917590:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000001c023af. 00000100:00000040:2.0:1784949268.917594:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f230fea2 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:174/0 lens 224/0 e 0 to 0 dl 1784949279 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949268.917596:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046e93177 x1871651289685888/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:174/0 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949268.917598:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000516cc351 00000400:00000010:3.0:1784949268.917603:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000516cc351. 00000100:00000001:3.0:1784949268.917609:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949268.917616:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.917630:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:1.0:1784949268.917635:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000046e93177 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289685888:12345-0@lo:400: Request processed in 530us (992us total) trans 0 rc 0/0 00000020:00000010:2.0:1784949268.917636:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000200:3.0:1784949268.917640:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.917642:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949268.917650:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00100000:1.0:1784949268.917655:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 815 00000100:00100000:2.0:1784949268.917662:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f230fea2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289686016:12345-192.168.202.137@tcp:400: 00000100:00000040:1.0:1784949268.917664:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949268.917664:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:2.0:1784949268.917666:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686016 00000100:00000001:3.0:1784949268.917669:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949268.917670:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:2.0:1784949268.917671:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949268.917673:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:1.0:1784949268.917675:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:2.0:1784949268.917676:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:3.0:1784949268.917677:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:2.0:1784949268.917680:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949268.917681:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000319b9e18. 00000100:00000001:3.0:1784949268.917681:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949268.917685:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949268.917688:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:3.0:1784949268.917689:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949268.917690:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000001:2.0:1784949268.917691:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000010:1.0:1784949268.917697:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000020:00000001:2.0:1784949268.917697:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949268.917699:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784949268.917699:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=140 v=5 (5 5 5 5) 00000020:00000040:1.0:1784949268.917704:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:2.0:1784949268.917705:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00001000:3.0:1784949268.917706:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=140 v=5 (1 1 1 1) 00000020:00000001:2.0:1784949268.917709:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:1.0:1784949268.917712:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:2.0:1784949268.917712:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949268.917713:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:1.0:1784949268.917715:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000017ac11c7. 00000020:00000001:2.0:1784949268.917718:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949268.917718:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949268.917719:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:3.0:1784949268.917764:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:1.0:1784949268.917766:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.917767:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00010000:00000001:3.0:1784949268.917769:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949268.917775:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:3.0:1784949268.917779:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949268.917780:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000010:2.0:1784949268.917782:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000008e253df. 02000000:00000001:2.0:1784949268.917806:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949268.917809:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eadc14f. 00000100:00000040:3.0:1784949268.917809:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.917811:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949268.917816:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005606381f. 00000020:00000001:2.0:1784949268.917819:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949268.917826:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686016 00000100:00000001:3.0:1784949268.917828:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784949268.917832:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949268.917833:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949268.917836:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:2.0:1784949268.917843:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f230fea2 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:174/0 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949268.917843:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949268.917876:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289685888:0@lo:400: 00010000:00000001:2.0:1784949268.917878:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949268.917881:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949268.917886:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=135 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949268.917888:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.917891:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949268.917892:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949268.917896:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000040:3.0:1784949268.917897:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000046cb422a x1871651289685888/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.917901:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949268.917906:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949268.917909:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.917914:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949268.917915:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:2.0:1784949268.917919:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949268.917938:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949268.917942:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 02000000:00000001:3.0:1784949268.917947:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000010:2.0:1784949268.917948:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000020:00000001:3.0:1784949268.917951:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000200:2.0:1784949268.917954:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289686016, offset 224 00000020:00000040:3.0:1784949268.917954:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949268.917958:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949268.917961:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 00000400:00000200:2.0:1784949268.917964:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949268.917966:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949268.917969:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949268.917970:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000046cb422a. 00000400:00000200:2.0:1784949268.917974:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949268.917976:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949268.917979:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949268.917983:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3841:3841:232:4294967295] 192.168.202.137@tcp LPNI seq info [2449:2449:8:4294967295] 00000100:00000001:3.0:1784949268.917983:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949268.917990:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949268.917994:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949268.918019:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949268.918019:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949268.918026:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949268.918035:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949268.918040:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c2b2d4f6. 00000800:00000200:2.0:1784949268.918048:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949268.918055:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949268.918059:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c2b2d4f6 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949268.918075:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949268.918078:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949268.918081:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949268.918083:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949268.918085:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949268.918090:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f230fea2 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:174/0 lens 224/224 e 0 to 0 dl 1784949279 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949268.918102:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f230fea2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289686016:12345-192.168.202.137@tcp:400: Request processed in 470us (856us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949268.918115:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 791 00000100:00000040:2.0:1784949268.918119:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949268.918121:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949268.918124:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949268.918129:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000015bbcec8. 00000020:00000010:2.0:1784949268.918133:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949268.918137:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000059e5b358. 00000020:00000040:2.0:1784949268.918141:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949268.918146:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949268.918252:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949268.918260:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c2b2d4f6. 00000800:00000001:3.0:1784949268.918263:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949268.918268:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949268.918276:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949268.918284:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949268.918285:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949268.918289:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949268.918299:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000200:3.0:1784949268.918306:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000010:1.0:1784949268.918307:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000100:00000001:1.0:1784949268.918317:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949268.918321:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949268.918329:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e000 00000400:00000200:3.0:1784949268.918338:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x94b9 [1] + 224 00000800:00000001:3.0:1784949268.918346:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949268.918359:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949268.918362:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949268.918366:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949268.918371:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949268.918375:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949268.918386:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949268.918408:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949268.918416:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949268.918418:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.918462:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949268.918465:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949268.918468:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949268.918469:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949268.918471:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000ef44eaa. 00000400:00000200:2.0:1784949268.918475:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784949268.918476:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:2.0:1784949268.918478:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949268.918482:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949268.918489:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.918494:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949268.918495:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.918497:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949268.918498:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949268.918499:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949268.918500:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949268.918501:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949268.918504:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=135 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949268.918506:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=135 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949268.918508:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949268.918509:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949268.918510:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949268.918513:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949268.918518:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.918520:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949268.918521:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949268.918522:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949268.918523:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949268.918524:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.918529:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949268.918530:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949268.918532:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949268.918533:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949268.918538:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006d868603 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289686016:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949268.918540:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.918542:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949268.918543:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006d868603 x1871651289686016/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949284 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949268.918548:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949268.918549:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949268.918551:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000032fe57fd. 02000000:00000001:2.0:1784949268.918553:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949268.918554:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949268.918555:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949268.918556:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949268.918557:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 02000000:00000001:2.0:1784949268.918559:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949268.918560:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949268.918561:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006d868603. 00000100:00000001:2.0:1784949268.918572:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949268.918573:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.918574:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949268.918577:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949268.918578:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949268.918579:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949268.918580:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949268.958956:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306208298, at 4306208299 00000100:00000010:0.0:1784949268.958981:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:1.0:1784949268.958981:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306208298, at 4306208299 00000100:00000010:1.0:1784949268.958997:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:2.0:1784949268.959958:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306208298, at 4306208300 00000100:00000010:2.0:1784949268.959965:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000040:3.0:1784949268.959970:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306208299, at 4306208300 00000100:00000010:3.0:1784949268.959979:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000800:00000100:2.0:1784949269.108181:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.4@tcp 00000800:00000010:2.0:1784949269.108188:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c53910d9. 00000400:00000200:2.0:1784949269.108196:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949269.108202:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949269.108210:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11542 health :900 00000400:00000200:2.0:1784949269.108217:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949269.108221:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:2.0:1784949269.108226:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:1.0:1784949269.108311:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949269.108323:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2861 00000400:00000001:1.0:1784949269.108332:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949269.108334:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949269.108337:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:1.0:1784949269.108339:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000400:00000200:1.0:1784949269.108345:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949269.108350:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:1.0:1784949269.108354:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000dcf1bf81. 00000400:00000200:1.0:1784949269.108361:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-110 00000400:00000200:1.0:1784949269.108364:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -110 00000400:00000200:1.0:1784949269.108368:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00010000:00000001:2.0:1784949269.747981:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949269.747990:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949269.747997:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949269.747999:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949269.748080:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949269.748088:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949269.748092:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949269.940134:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949269.940150:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949269.940153:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949269.940161:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949269.940165:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949269.940170:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949269.940173:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949270.132075:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:2.0:1784949270.132080:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.10@tcp 00000400:00000010:3.0:1784949270.132087:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000007028fecb. 00000800:00000010:2.0:1784949270.132089:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000400:00000200:2.0:1784949270.132104:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:3.0:1784949270.132105:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000009f047fe3. 00000400:00000010:3.0:1784949270.132116:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003dcd3ddc. 00000400:00000200:2.0:1784949270.132117:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949270.132133:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11543 health :800 00000400:00000200:2.0:1784949270.132140:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:3.0:1784949270.132145:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000004fd5390b. 00000400:00000200:2.0:1784949270.132146:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:2.0:1784949270.132164:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:3.0:1784949270.132166:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949270.132201:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949270.132210:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949270.132215:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3842:3842:233:4294967295] 192.168.202.137@tcp LPNI seq info [2450:2450:8:4294967295] 00000400:00000200:3.0:1784949270.132242:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949270.132257:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949270.132264:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949270.132271:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003547cd5f. 00000400:00000200:1.0:1784949270.132276:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:3.0:1784949270.132286:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949270.132296:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2861 00000800:00000200:3.0:1784949270.132308:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:1.0:1784949270.132313:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:3.0:1784949270.132315:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 2, nob 96 niov 1 nkiov 0 00000400:00000001:1.0:1784949270.132323:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949270.132332:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:1.0:1784949270.132338:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000400:00000200:1.0:1784949270.132349:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949270.132359:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:1.0:1784949270.132369:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b1cbf66f. 00000400:00000200:1.0:1784949270.132380:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-110 00000400:00000200:1.0:1784949270.132385:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -110 00000400:00000200:1.0:1784949270.132393:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00000800:00000200:0.0:1784949270.132574:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949270.132582:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003547cd5f. 00000400:00000200:0.0:1784949270.132590:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949270.132597:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949270.132603:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949270.132607:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784949270.132621:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949270.132635:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949270.132639:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949270.132646:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949270.132656:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949270.132675:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949270.132692:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949270.132708:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949270.132753:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3843:3843:234:4294967295] 192.168.202.137@tcp LPNI seq info [2451:2451:8:4294967295] 00000400:00000200:0.0:1784949270.132771:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949270.132780:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949270.132788:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000800:00000200:0.0:1784949270.132797:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949270.132807:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949270.132812:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949270.132943:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949270.132972:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949270.132978:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000400:00000200:0.0:1784949270.132986:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949270.132992:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949270.132993:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949270.133003:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949270.133016:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949270.133025:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949270.133034:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949270.133049:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784949270.133070:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x94cd 00000800:00000001:1.0:1784949270.133079:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949270.133094:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949270.133099:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949270.133106:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949270.133115:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003dcd3ddc 00000400:00000010:1.0:1784949270.133120:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003dcd3ddc. 00000400:00000200:1.0:1784949270.133125:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949270.133131:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000007028fecb. 00000400:00000010:1.0:1784949270.133137:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000009f047fe3. 00000800:00000001:1.0:1784949270.133149:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949270.133154:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949270.772030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949270.772040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949270.772048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949270.772049:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949270.772163:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949270.772171:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949270.772176:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949270.964020:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949270.964036:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949270.964039:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949270.964046:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949270.964049:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949270.964054:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949270.964057:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1784949271.156221:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.8@tcp 00000800:00000010:2.0:1784949271.156232:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cf155122. 00000400:00000200:2.0:1784949271.156242:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949271.156250:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949271.156259:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11544 health :900 00000400:00000200:2.0:1784949271.156268:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949271.156275:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:2.0:1784949271.156281:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000800:00000100:2.0:1784949271.156317:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.14@tcp 00000800:00000010:2.0:1784949271.156322:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000400:00000200:2.0:1784949271.156327:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949271.156332:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949271.156336:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949271.156340:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:2.0:1784949271.156345:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:1.0:1784949271.156409:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949271.156422:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2861 00000400:00000001:1.0:1784949271.156432:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949271.156435:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949271.156439:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000010:1.0:1784949271.156442:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000400:00000200:1.0:1784949271.156448:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949271.156453:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:1.0:1784949271.156458:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007db69e48. 00000400:00000200:1.0:1784949271.156466:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-110 00000400:00000200:1.0:1784949271.156470:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -110 00000400:00000200:1.0:1784949271.156474:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000400:00000200:1.0:1784949271.156479:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2861 00000400:00000001:1.0:1784949271.156482:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949271.156484:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949271.156486:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000010:1.0:1784949271.156489:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000400:00000200:1.0:1784949271.156491:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949271.156494:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:1.0:1784949271.156498:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004215ddb7. 00000400:00000200:1.0:1784949271.156502:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-110 00000400:00000200:1.0:1784949271.156504:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -110 00000400:00000200:1.0:1784949271.156507:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00010000:00000001:2.0:1784949271.796040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949271.796048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949271.796055:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949271.796059:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949271.796063:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949271.796066:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949271.796071:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949271.796074:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949271.796077:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949271.796079:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949271.796227:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949271.796238:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949271.796244:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949271.988025:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949271.988042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949271.988046:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949271.988052:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949271.988057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949271.988062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949271.988067:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949272.180045:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949272.180056:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000af276f2a. 00000400:00000010:3.0:1784949272.180068:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000378c7327. 00000400:00000010:3.0:1784949272.180076:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000752180c2. 00000400:00000010:3.0:1784949272.180087:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001ff78f22. 00000400:00000200:3.0:1784949272.180093:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949272.180113:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949272.180123:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949272.180128:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3844:3844:236:4294967295] 192.168.202.137@tcp LPNI seq info [2452:2452:8:4294967295] 00000400:00000200:3.0:1784949272.180137:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949272.180146:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949272.180154:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949272.180159:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000800:00000200:3.0:1784949272.180168:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949272.180178:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949272.180183:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949272.180215:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11546 health :800 00000800:00000200:2.0:1784949272.180408:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949272.180417:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000400:00000200:2.0:1784949272.180429:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949272.180444:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:2.0:1784949272.180455:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:0.0:1784949272.180461:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949272.180463:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:0.0:1784949272.180488:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949272.180497:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949272.180506:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949272.180520:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949272.180541:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949272.180555:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949272.180571:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949272.180578:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3845:3845:236:4294967295] 192.168.202.137@tcp LPNI seq info [2453:2453:8:4294967295] 00000400:00000200:0.0:1784949272.180594:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949272.180603:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949272.180609:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000800:00000200:0.0:1784949272.180617:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949272.180626:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949272.180631:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949272.180773:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949272.180837:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949272.180845:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000400:00000200:0.0:1784949272.180853:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949272.180863:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949272.180865:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949272.180883:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949272.180886:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949272.180893:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949272.180945:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949272.180960:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949272.180983:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x94d1 00000800:00000001:2.0:1784949272.180992:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949272.181006:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949272.181011:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949272.181018:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949272.181027:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000752180c2 00000400:00000010:2.0:1784949272.181033:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000752180c2. 00000400:00000200:2.0:1784949272.181039:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949272.181044:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000af276f2a. 00000400:00000010:2.0:1784949272.181051:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000378c7327. 00000800:00000001:2.0:1784949272.181061:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949272.181066:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949272.820008:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949272.820017:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949272.820024:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949272.820025:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949272.820132:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949272.820137:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949272.820141:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949273.012045:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949273.012059:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949273.012062:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949273.012069:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949273.012074:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949273.012078:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949273.012083:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949273.844041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949273.844051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949273.844059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949273.844061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949273.844205:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949273.844212:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949273.844216:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1784949274.036072:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949274.036073:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949274.036087:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949274.036089:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949274.036094:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1784949274.036097:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046cb422a. 00000100:00000001:1.0:1784949274.036100:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949274.036105:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1784949274.036110:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949274.036112:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949274.036123:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949274.036126:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949274.036132:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949274.036140:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949274.036153:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00000001:3.0:1784949274.036164:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949274.036178:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949274.036194:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949274.036197:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949274.036206:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046cb422a] to pc [ptlrpcd_00_01+1] req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949274.036250:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949274.036254:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949274.036259:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949274.036263:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949274.036267:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949274.036272:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000020:00000040:3.0:1784949274.036276:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949274.036280:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949274.036283:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949274.036285:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949274.036291:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949274.036295:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949274.036314:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949274.036319:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949274.036325:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949274.036328:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949274.036334:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949274.036337:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949274.036338:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949274.036339:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949274.036350:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949274.036356:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_00_02+2] req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949274.036395:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949274.036400:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949274.036408:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949274.036412:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949274.036416:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949274.036417:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949274.036424:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289686144:0@lo:400: 00000100:00000040:3.0:1784949274.036430:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11551) 00000100:00000001:1.0:1784949274.036434:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949274.036445:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949274.036446:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949274.036449:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949274.036456:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949274.036456:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949274.036465:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1784949274.036469:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000229221cc. 00000100:00000001:3.0:1784949274.036473:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949274.036475:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949274.036491:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949274.036494:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949274.036498:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000877cac20. 00000400:00000010:1.0:1784949274.036515:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000100:00000001:2.0:1784949274.036521:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949274.036526:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949274.036528:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686144, portal 25 00000100:00000001:1.0:1784949274.036534:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1784949274.036534:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949274.036542:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554953472 : -115544154598144 : ffff96e9c7b59500) 02000000:00000001:2.0:1784949274.036543:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949274.036552:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949274.036554:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949274.036568:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289686272:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949274.036573:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949274.036592:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949274.036593:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:2.0:1784949274.036602:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:1.0:1784949274.036603:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 02000000:00000001:2.0:1784949274.036610:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949274.036612:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289686144, offset 0 02000000:00000001:2.0:1784949274.036622:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949274.036629:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:2.0:1784949274.036635:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000059e5b358. 02000000:00000001:2.0:1784949274.036643:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949274.036653:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:2.0:1784949274.036658:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000ef44eaa. 00000400:00000010:2.0:1784949274.036673:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000200:1.0:1784949274.036674:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e080 00000100:00000200:2.0:1784949274.036715:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686272, portal 10 00000100:00000001:2.0:1784949274.036723:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949274.036739:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8ac1 [2] + 0 00000100:00000001:2.0:1784949274.036739:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554960640 : -115544154590976 : ffff96e9c7b5b100) 00000400:00000010:1.0:1784949274.036755:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000cc579134. 00000100:00000040:2.0:1784949274.036764:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949274.036776:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000f9492b64 00000100:00000001:2.0:1784949274.036787:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949274.036797:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949274.036798:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1784949274.036805:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000200:1.0:1784949274.036815:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:2.0:1784949274.036817:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289686272, offset 0 00000400:00000200:1.0:1784949274.036830:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f9492b64 00000400:00000010:1.0:1784949274.036838:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000f9492b64. 00000100:00000001:1.0:1784949274.036848:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949274.036852:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1784949274.036856:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949274.036861:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000470139eb x1871651289686144 msgsize 224 00000400:00000200:2.0:1784949274.036881:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00100000:1.0:1784949274.036882:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949274.036893:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:2.0:1784949274.036900:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949274.036971:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949274.036973:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949274.036985:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949274.036993:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949274.037004:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3846:3846:236:4294967295] 192.168.202.137@tcp LPNI seq info [2454:2454:8:4294967295] 00000100:00000001:0.0:1784949274.037011:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949274.037021:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686144 02000000:00000001:0.0:1784949274.037030:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949274.037038:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784949274.037040:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949274.037047:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949274.037057:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949274.037059:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949274.037062:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784949274.037070:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00100000:0.0:1784949274.037072:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686144 00000100:00000001:1.0:1784949274.037078:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000010:2.0:1784949274.037078:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000020:00000001:0.0:1784949274.037082:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:1.0:1784949274.037090:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1784949274.037091:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949274.037102:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:2.0:1784949274.037105:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000040:0.0:1784949274.037113:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784949274.037128:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:2.0:1784949274.037131:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949274.037132:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784949274.037141:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:1.0:1784949274.037147:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949274.037147:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949274.037153:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949274.037167:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:1.0:1784949274.037169:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949274.037177:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949274.037180:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949274.037186:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949274.037190:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000001:1.0:1784949274.037191:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949274.037200:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2238 usecs (3 jiffies) 00000100:00000001:1.0:1784949274.037205:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306213419 00000100:00000001:2.0:1784949274.037210:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306213380 00000020:00000010:0.0:1784949274.037211:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b465f54. 00000100:00000010:1.0:1784949274.037215:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000020:00000010:0.0:1784949274.037230:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000bb50d8a2. 00000100:00000001:1.0:1784949274.037231:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949274.037244:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306213419 00000020:00000010:0.0:1784949274.037247:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002ffa19fa. 00000100:00000001:2.0:1784949274.037247:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306213419 00000100:00000010:1.0:1784949274.037254:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b3a6ec08. 00000100:00000001:2.0:1784949274.037260:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306213380 00000100:00000001:1.0:1784949274.037263:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1784949274.037273:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949274.037275:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306213419 00000100:00000001:2.0:1784949274.037276:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306213419 00000100:00000001:0.0:1784949274.037286:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:1.0:1784949274.037289:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000001:2.0:1784949274.037289:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306213380 00000100:00000001:0.0:1784949274.037293:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949274.037299:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949274.037303:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949274.037307:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306213419 00000100:00000001:2.0:1784949274.037308:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306213419 00000100:00000010:1.0:1784949274.037317:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000100:00000001:2.0:1784949274.037319:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306213380 00000100:00000001:0.0:1784949274.037325:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949274.037326:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949274.037338:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949274.037339:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306213419 00000100:00000001:1.0:1784949274.037344:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.037349:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949274.037351:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949274.037356:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949274.037358:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949274.037363:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949274.037364:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949274.037370:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000100:00000001:1.0:1784949274.037377:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949274.037378:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949274.037382:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949274.037384:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949274.037389:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949274.037389:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949274.037395:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949274.037399:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949274.037402:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949274.037403:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949274.037406:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949274.037411:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.037415:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1784949274.037418:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 816 00000400:00000200:3.0:1784949274.037420:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000100:00000001:1.0:1784949274.037422:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.037427:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949274.037428:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000040:0.0:1784949274.037437:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949274.037438:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949274.037439:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949274.037448:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949274.037450:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818134096 : -115538891417520 : ffff96eb016b5850) 00000800:00000001:1.0:1784949274.037451:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784949274.037455:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949274.037463:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949274.037480:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949274.037483:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000470139eb x1871651289686144/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:180/0 lens 224/0 e 0 to 0 dl 1784949285 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949274.037485:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949274.037486:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949274.037498:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949274.037499:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949274.037523:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949274.037552:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784949274.037552:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e100 00000100:00000001:3.0:1784949274.037553:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949274.037562:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:0.0:1784949274.037565:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000400:00000200:1.0:1784949274.037586:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 178448 00000100:00000001:0.0:1784949274.037587:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949274.037600:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000800:00000001:1.0:1784949274.037602:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949274.037613:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949274.037615:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000470139eb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289686144:12345-0@lo:400: 00000400:00000200:1.0:1784949274.037620:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1784949274.037633:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686144 00000400:00000200:1.0:1784949274.037635:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784949274.037645:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949274.037648:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949274.037654:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:0.0:1784949274.037657:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000010:1.0:1784949274.037661:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ee628dd6. 00000020:00000001:0.0:1784949274.037665:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949274.037668:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000ee628dd6 x1871651289686272 msgsize 224 00000020:00000001:0.0:1784949274.037675:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00100000:1.0:1784949274.037707:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:0.0:1784949274.037711:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949274.037730:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949274.037734:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949274.037740:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949274.037744:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949274.037744:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:0.0:1784949274.037754:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949274.037758:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:1.0:1784949274.037760:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949274.037765:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000001:1.0:1784949274.037769:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949274.037773:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949274.037785:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000efbd7ee7. 02000000:00000001:0.0:1784949274.037792:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949274.037797:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949274.037803:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949274.037808:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686144 00010000:00000001:0.0:1784949274.037814:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949274.037824:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000470139eb x1871651289686144/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:180/0 lens 224/224 e 0 to 0 dl 1784949285 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949274.037825:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949274.037836:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686272 02000000:00000001:2.0:1784949274.037845:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949274.037853:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00010000:00000001:0.0:1784949274.037856:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949274.037861:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949274.037864:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949274.037871:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949274.037875:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=146 v=5 (1 1 1 1) 00000100:00100000:2.0:1784949274.037881:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686272 00000100:00000001:0.0:1784949274.037887:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1784949274.037890:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949274.037897:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000040:0.0:1784949274.037900:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949274.037967:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000001:2.0:1784949274.037968:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949274.037978:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000001:0.0:1784949274.037979:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949274.037985:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949274.037990:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949274.037994:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1784949274.037999:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:0.0:1784949274.038009:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949274.038013:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:0.0:1784949274.038020:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000020:00000010:2.0:1784949274.038024:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000200:0.0:1784949274.038035:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289686144, offset 224 00000100:00000001:2.0:1784949274.038036:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949274.038048:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000400:00000200:0.0:1784949274.038058:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:2.0:1784949274.038064:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b1cbf66f. 00000020:00000010:2.0:1784949274.038077:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085067f70. 00000400:00000200:0.0:1784949274.038078:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:2.0:1784949274.038090:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000200:0.0:1784949274.038103:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e080 00000100:00000040:2.0:1784949274.038107:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949274.038115:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:0.0:1784949274.038117:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x94d5 [1] + 224 00000100:00000001:2.0:1784949274.038121:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949274.038129:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949274.038137:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949274.038141:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:0.0:1784949274.038151:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949274.038152:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949274.038159:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949274.038166:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949274.038177:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949274.038185:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949274.038202:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949274.038208:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784949274.038210:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949274.038225:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 792 00000100:00000001:1.0:1784949274.038233:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:2.0:1784949274.038238:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949274.038241:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949274.038247:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.038250:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637816320 : -115544071735296 : ffff96e9cca5f800) 00000100:00000001:1.0:1784949274.038254:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949274.038270:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ee628dd6 x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:180/0 lens 224/0 e 0 to 0 dl 1784949285 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949274.038276:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949274.038292:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00000400:00000010:0.0:1784949274.038300:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000100:00000001:0.0:1784949274.038311:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949274.038317:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949274.038324:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949274.038328:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949274.038333:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000040:0.0:1784949274.038339:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784949274.038346:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1784949274.038349:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949274.038357:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949274.038358:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000020:00000001:0.0:1784949274.038369:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00100000:2.0:1784949274.038379:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ee628dd6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289686272:12345-192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949274.038382:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949274.038385:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000470139eb x1871651289686144/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:180/0 lens 224/224 e 0 to 0 dl 1784949285 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949274.038389:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:2.0:1784949274.038399:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686272 00000400:00000001:3.0:1784949274.038402:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949274.038411:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1784949274.038413:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:3.0:1784949274.038417:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000877cac20. 00000020:00000001:2.0:1784949274.038424:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:3.0:1784949274.038428:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000100:00100000:0.0:1784949274.038434:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000470139eb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289686144:12345-0@lo:400: Request processed in 875us (1564us total) trans 0 rc 0/0 00000020:00000001:2.0:1784949274.038435:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949274.038435:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000100:00000001:3.0:1784949274.038446:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1784949274.038449:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949274.038456:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000200:3.0:1784949274.038460:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949274.038469:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949274.038487:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949274.038500:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00100000:0.0:1784949274.038504:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 816 00000020:00000001:2.0:1784949274.038508:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949274.038516:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949274.038521:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:2.0:1784949274.038524:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000200:3.0:1784949274.038528:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949274.038534:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949274.038535:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949274.038546:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949274.038548:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949274.038549:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:2.0:1784949274.038553:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000c3dbeb98. 00000100:00000001:3.0:1784949274.038557:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949274.038560:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949274.038562:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000bb50d8a2. 00000100:00000001:3.0:1784949274.038563:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:2.0:1784949274.038567:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949274.038571:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:2.0:1784949274.038576:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949274.038577:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:0.0:1784949274.038578:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002ffa19fa. 00000100:00000001:3.0:1784949274.038584:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949274.038586:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686272 00000020:00000010:0.0:1784949274.038592:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004b465f54. 02000000:00000001:3.0:1784949274.038596:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949274.038597:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00001000:3.0:1784949274.038609:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=146 v=5 (5 5 5 5) 00000020:00000040:0.0:1784949274.038610:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000200:2.0:1784949274.038610:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ee628dd6 x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:180/0 lens 224/224 e 0 to 0 dl 1784949285 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949274.038627:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:3.0:1784949274.038628:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=146 v=5 (1 1 1 1) 02000000:00000010:0.0:1784949274.038636:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000efbd7ee7. 00010000:00000001:2.0:1784949274.038641:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949274.038644:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000001:0.0:1784949274.038649:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:2.0:1784949274.038651:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949274.038651:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949274.038659:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949274.038665:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00001000:2.0:1784949274.038666:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=141 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949274.038671:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:3.0:1784949274.038673:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.038705:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949274.038716:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949274.038716:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949274.038721:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a6862d86. 00000100:00000001:2.0:1784949274.038729:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000040:3.0:1784949274.038733:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949274.038736:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006e611703. 02000000:00000001:2.0:1784949274.038737:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949274.038743:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.038753:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949274.038758:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949274.038766:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949274.038769:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:2.0:1784949274.038775:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000100:00000001:3.0:1784949274.038776:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949274.038790:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289686272, offset 224 00000100:00000040:3.0:1784949274.038791:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949274.038815:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000046cb422a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289686144:0@lo:400: 00000400:00000200:2.0:1784949274.038816:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949274.038827:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949274.038831:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949274.038836:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000046cb422a x1871651289686144/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949274.038847:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949274.038854:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949274.038859:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949274.038865:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000229221cc. 00000400:00000200:2.0:1784949274.038869:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3847:3847:236:4294967295] 192.168.202.137@tcp LPNI seq info [2455:2455:8:4294967295] 02000000:00000001:3.0:1784949274.038872:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949274.038878:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949274.038882:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949274.038888:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949274.038893:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 02000000:00000001:3.0:1784949274.038901:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:2.0:1784949274.038942:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:3.0:1784949274.038946:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949274.038953:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000046cb422a. 00000800:00000200:2.0:1784949274.038959:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949274.038961:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949274.038967:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949274.038970:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000100:00000001:3.0:1784949274.038971:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949274.038986:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:2.0:1784949274.038987:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949274.038992:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949274.038998:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1784949274.039004:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784949274.039004:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949274.039013:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949274.039028:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949274.039033:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949274.039036:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949274.039039:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949274.039043:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949274.039050:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ee628dd6 x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:180/0 lens 224/224 e 0 to 0 dl 1784949285 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949274.039062:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ee628dd6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289686272:12345-192.168.202.137@tcp:400: Request processed in 739us (1392us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949274.039072:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 792 00000100:00000040:2.0:1784949274.039077:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949274.039080:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949274.039083:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949274.039088:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085067f70. 00000020:00000010:2.0:1784949274.039093:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949274.039097:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b1cbf66f. 00000020:00000040:2.0:1784949274.039102:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949274.039105:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949274.039170:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949274.039176:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000400:00000200:3.0:1784949274.039182:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949274.039191:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949274.039197:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000400:00000010:3.0:1784949274.039200:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000100:00000001:3.0:1784949274.039204:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949274.039206:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949274.039215:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949274.039223:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949274.039226:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949274.039230:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949274.039237:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949274.039249:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e100 00000400:00000200:3.0:1784949274.039255:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x94dd [1] + 224 00000800:00000001:3.0:1784949274.039262:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949274.039268:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949274.039271:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949274.039275:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949274.039278:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949274.039282:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949274.039293:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949274.039316:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949274.039323:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949274.039326:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949274.039368:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949274.039373:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949274.039378:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949274.039381:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949274.039383:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000ef44eaa. 00000400:00000200:2.0:1784949274.039390:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784949274.039392:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:2.0:1784949274.039395:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949274.039401:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949274.039413:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949274.039422:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949274.039424:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949274.039426:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949274.039428:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949274.039430:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949274.039432:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949274.039434:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949274.039440:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=141 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949274.039444:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=141 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949274.039447:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949274.039448:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949274.039451:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949274.039455:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949274.039463:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.039467:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949274.039469:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949274.039471:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949274.039473:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949274.039477:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949274.039485:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949274.039487:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949274.039489:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949274.039492:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949274.039500:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289686272:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949274.039505:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949274.039507:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949274.039509:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289686272/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949274.039517:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949274.039518:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949274.039520:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000059e5b358. 02000000:00000001:2.0:1784949274.039523:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949274.039525:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949274.039526:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949274.039529:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949274.039530:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 02000000:00000001:2.0:1784949274.039534:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949274.039535:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949274.039537:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000100:00000001:2.0:1784949274.039541:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949274.039542:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949274.039544:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949274.039549:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949274.039551:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949274.039553:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949274.039555:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949274.080019:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306213419, at 4306213420 00000100:00000040:2.0:1784949274.080029:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306213419, at 4306213420 00000100:00000040:3.0:1784949274.080032:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306213419, at 4306213420 00000100:00000010:0.0:1784949274.080034:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:1.0:1784949274.080045:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306213419, at 4306213420 00000100:00000010:2.0:1784949274.080045:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:3.0:1784949274.080051:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00000100:00000010:1.0:1784949274.080065:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b3a6ec08. 00010000:00000001:2.0:1784949274.868016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949274.868026:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949274.868034:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949274.868036:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949274.868155:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949274.868162:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949274.868166:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949275.060006:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949275.060021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949275.060024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949275.060030:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949275.060033:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949275.060038:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949275.060042:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949275.892050:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949275.892059:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949275.892067:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949275.892069:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949275.892215:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949275.892221:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949275.892226:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949276.084025:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949276.084039:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949276.084041:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949276.084046:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949276.084049:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949276.084054:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949276.084057:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949276.916046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949276.916055:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949276.916063:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949276.916065:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949276.916191:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949276.916198:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949276.916203:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949277.108054:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949277.108070:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949277.108073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949277.108079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949277.108083:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949277.108087:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949277.108091:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949277.940013:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949277.940023:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949277.940031:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949277.940033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949277.940163:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949277.940170:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949277.940175:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949278.132080:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949278.132099:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949278.132102:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949278.132109:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949278.132113:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949278.132117:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949278.132121:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949278.964046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949278.964056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949278.964064:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949278.964066:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949278.964191:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949278.964198:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949278.964202:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949279.156058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949279.156073:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949279.156075:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949279.156081:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949279.156085:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1784949279.156086:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949279.156090:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.156101:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.156103:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949279.156116:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004f010ef0. 00000020:00000040:2.0:1784949279.156125:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949279.156132:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949279.156138:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949279.156146:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949279.156158:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000004c284093. 00000100:00000001:2.0:1784949279.156167:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949279.156181:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949279.156199:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949279.156202:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949279.156210:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004f010ef0] to pc [ptlrpcd_00_03+3] req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949279.156249:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949279.156254:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949279.156260:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949279.156264:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949279.156270:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949279.156277:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009e01665e. 00000020:00000040:2.0:1784949279.156281:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949279.156284:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949279.156286:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949279.156289:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.156290:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1784949279.156297:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000ffbbc320. 00000100:00000001:3.0:1784949279.156298:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949279.156305:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949279.156307:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.156311:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949279.156319:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949279.156320:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949279.156322:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949279.156373:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949279.156375:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949279.156378:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.156380:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949279.156386:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1784949279.156392:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e01665e] to pc [ptlrpcd_00_00+0] req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949279.156394:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949279.156405:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949279.156418:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004f010ef0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289686400:0@lo:400: 00000100:00000001:3.0:1784949279.156427:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949279.156435:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949279.156439:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949279.156443:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949279.156451:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949279.156452:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 02000000:00000001:3.0:1784949279.156461:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949279.156465:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11556) 00000400:00000010:3.0:1784949279.156473:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002fbcdf42. 00000400:00000010:3.0:1784949279.156484:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 00000100:00000001:2.0:1784949279.156490:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949279.156492:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1784949279.156495:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686400, portal 25 00000100:00000001:2.0:1784949279.156501:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.156502:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949279.156507:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949279.156510:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949279.156512:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469584512 : -115544239967104 : ffff96e9c29ef480) 00000100:00000040:0.0:1784949279.156525:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949279.156526:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949279.156589:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949279.156590:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949279.156596:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949279.156605:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784949279.156606:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949279.156613:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949279.156618:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949279.156621:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000100:00100000:0.0:1784949279.156625:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000009e01665e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289686528:192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949279.156635:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289686400, offset 0 00000100:00000001:0.0:1784949279.156636:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949279.156668:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949279.156674:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949279.156682:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1784949279.156683:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:0.0:1784949279.156690:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 02000000:00000001:0.0:1784949279.156696:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949279.156703:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005bb236bd. 00000400:00000010:0.0:1784949279.156711:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fcd1fd6e. 00000400:00000200:3.0:1784949279.156713:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:0.0:1784949279.156720:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686528, portal 10 00000100:00000001:0.0:1784949279.156727:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949279.156729:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469578240 : -115544239973376 : ffff96e9c29edc00) 00000400:00000200:3.0:1784949279.156741:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e180 00000100:00000040:0.0:1784949279.156742:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949279.156762:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8ae5 [2] + 0 00000100:00000001:0.0:1784949279.156765:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:3.0:1784949279.156775:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000100:00000040:0.0:1784949279.156777:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949279.156787:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00000400:00000200:3.0:1784949279.156788:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c9198cec 00000100:00000200:0.0:1784949279.156798:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289686528, offset 0 00000400:00000200:3.0:1784949279.156805:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949279.156811:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949279.156819:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949279.156832:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949279.156842:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1784949279.156851:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949279.156857:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949279.156863:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949279.156869:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3848:3848:236:4294967295] 192.168.202.137@tcp LPNI seq info [2456:2456:8:4294967295] 00000400:00000200:0.0:1784949279.156888:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949279.156899:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949279.156901:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c9198cec 00000400:00000010:3.0:1784949279.156946:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000c9198cec. 00000100:00000001:3.0:1784949279.156956:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000800:00000010:0.0:1784949279.156958:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000100:00000200:3.0:1784949279.156960:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949279.156971:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000f3ed5d93 x1871651289686400 msgsize 224 00000800:00000200:0.0:1784949279.156980:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:3.0:1784949279.156992:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784949279.156997:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:0.0:1784949279.157005:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949279.157012:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949279.157023:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949279.157038:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949279.157040:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000400:00000010:3.0:1784949279.157047:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000100:00000040:0.0:1784949279.157048:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2241 usecs (3 jiffies) 00000100:00000001:3.0:1784949279.157055:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949279.157057:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306218500 00000100:00000200:3.0:1784949279.157066:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1784949279.157068:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000001:2.0:1784949279.157072:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949279.157081:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686400 00000100:00000001:0.0:1784949279.157087:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 02000000:00000001:2.0:1784949279.157090:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949279.157094:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949279.157097:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949279.157101:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306218500 00000100:00000040:3.0:1784949279.157104:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949279.157105:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949279.157114:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 02000000:00000001:2.0:1784949279.157117:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.157127:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00100000:2.0:1784949279.157129:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686400 00000100:00000001:3.0:1784949279.157138:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949279.157140:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949279.157142:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306218500 00000020:00000040:2.0:1784949279.157148:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949279.157149:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:0.0:1784949279.157153:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000020:00000001:2.0:1784949279.157157:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949279.157162:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949279.157163:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949279.157165:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949279.157175:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306218500 00000020:00000001:2.0:1784949279.157176:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949279.157177:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306218539 00000100:00000010:0.0:1784949279.157188:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000020:00000001:2.0:1784949279.157190:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949279.157200:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306218500 00000100:00000001:0.0:1784949279.157201:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949279.157202:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949279.157214:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:0.0:1784949279.157218:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949279.157219:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306218539 00000100:00000001:2.0:1784949279.157222:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949279.157231:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949279.157233:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000100:00000001:3.0:1784949279.157236:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306218500 00000100:00000001:0.0:1784949279.157239:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.157247:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949279.157248:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306218539 00000020:00000010:2.0:1784949279.157249:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000059e5b358. 00000100:00000001:0.0:1784949279.157255:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949279.157262:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:2.0:1784949279.157265:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085067f70. 00000100:00000001:3.0:1784949279.157266:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306218500 00000100:00000001:0.0:1784949279.157268:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949279.157278:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949279.157281:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306218539 00000020:00000010:2.0:1784949279.157282:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:0.0:1784949279.157283:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.157296:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.157297:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306218500 00000100:00000040:2.0:1784949279.157308:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949279.157310:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949279.157321:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949279.157321:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.157331:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949279.157332:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.157341:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.157342:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949279.157355:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949279.157360:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949279.157365:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.157365:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949279.157369:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949279.157377:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.157383:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.157396:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949279.157431:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949279.157436:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949279.157447:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000200:0.0:1784949279.157448:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00100000:2.0:1784949279.157449:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 817 00000100:00000001:3.0:1784949279.157455:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000010:0.0:1784949279.157458:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000100:00000040:2.0:1784949279.157468:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949279.157468:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949279.157470:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949279.157477:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.157483:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534818140240 : -115538891411376 : ffff96eb016b7050) 00000400:00000200:0.0:1784949279.157492:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949279.157501:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000100:00000040:2.0:1784949279.157502:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f3ed5d93 x1871651289686400/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:185/0 lens 224/0 e 0 to 0 dl 1784949290 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949279.157505:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000100:00000001:0.0:1784949279.157515:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949279.157525:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949279.157532:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949279.157541:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949279.157552:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007bca6f5b. 00000100:00000001:0.0:1784949279.157553:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949279.157557:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949279.157561:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949279.157563:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949279.157570:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784949279.157572:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007bca6f5b. 00000800:00000001:1.0:1784949279.157581:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949279.157591:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f3ed5d93 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289686400:12345-0@lo:400: 00000100:00000001:0.0:1784949279.157595:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.157605:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784949279.157608:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:2.0:1784949279.157609:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686400 00000020:00000001:2.0:1784949279.157618:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949279.157626:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949279.157629:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949279.157633:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e200 00000020:00000001:2.0:1784949279.157635:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949279.157661:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784949279.157671:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949279.157674:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949279.157678:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:1.0:1784949279.157680:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 178672 00000020:00000001:2.0:1784949279.157682:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949279.157689:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949279.157694:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:1.0:1784949279.157696:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949279.157698:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949279.157704:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:1.0:1784949279.157708:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784949279.157710:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008450ee2b. 00000400:00000200:1.0:1784949279.157715:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784949279.157717:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.157724:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949279.157728:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949279.157730:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949279.157739:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686400 00000100:00000001:1.0:1784949279.157742:0:337113:0:(events.c:286:request_in_callback()) Process entered 00010000:00000001:2.0:1784949279.157743:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000200:1.0:1784949279.157747:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:2.0:1784949279.157753:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f3ed5d93 x1871651289686400/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:185/0 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949279.157756:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009c295fd6. 00000100:00000040:1.0:1784949279.157765:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000009c295fd6 x1871651289686528 msgsize 224 00010000:00000001:2.0:1784949279.157777:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00100000:1.0:1784949279.157784:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000001:2.0:1784949279.157785:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949279.157794:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=151 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949279.157800:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949279.157804:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949279.157807:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:1.0:1784949279.157808:0:337113:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:2.0:1784949279.157814:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949279.157818:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.157822:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:1.0:1784949279.157824:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949279.157829:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:1.0:1784949279.157834:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949279.157838:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000200:2.0:1784949279.157847:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289686400, offset 224 00000400:00000200:2.0:1784949279.157857:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949279.157865:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784949279.157873:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00100000:0.0:1784949279.157874:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686528 02000000:00000001:0.0:1784949279.157880:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949279.157883:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949279.157887:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949279.157888:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e180 02000000:00000001:0.0:1784949279.157895:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949279.157903:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x94f1 [1] + 224 00000100:00100000:0.0:1784949279.157940:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686528 00000020:00000001:0.0:1784949279.157945:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949279.157947:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949279.157951:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949279.157955:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:0.0:1784949279.157963:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:0.0:1784949279.157970:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784949279.157971:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949279.157976:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784949279.157983:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:0.0:1784949279.157986:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000100:00000001:2.0:1784949279.157990:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949279.157994:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949279.157999:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00000200:2.0:1784949279.158000:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949279.158008:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000020:00000010:0.0:1784949279.158020:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000734334de. 00000100:00000040:2.0:1784949279.158025:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949279.158030:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002ffa19fa. 00000100:00000040:0.0:1784949279.158047:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949279.158051:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949279.158054:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949279.158055:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949279.158060:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949279.158067:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000100:00000001:0.0:1784949279.158070:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000010:2.0:1784949279.158073:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:0.0:1784949279.158076:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949279.158081:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.158082:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949279.158088:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949279.158096:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949279.158100:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784949279.158103:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949279.158105:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.158108:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949279.158109:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949279.158118:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f3ed5d93 x1871651289686400/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:185/0 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949279.158128:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949279.158133:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949279.158135:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f3ed5d93 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289686400:12345-0@lo:400: Request processed in 591us (1159us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949279.158148:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 793 00000100:00000040:0.0:1784949279.158156:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00100000:2.0:1784949279.158164:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 817 00000100:00000001:0.0:1784949279.158166:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637814528 : -115544071737088 : ffff96e9cca5f100) 00000100:00000040:2.0:1784949279.158175:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949279.158175:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949279.158181:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009c295fd6 x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:185/0 lens 224/0 e 0 to 0 dl 1784949290 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949279.158184:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949279.158186:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949279.158194:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000001:3.0:1784949279.158198:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:2.0:1784949279.158203:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085067f70. 00000400:00000001:3.0:1784949279.158205:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949279.158208:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:3.0:1784949279.158213:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002fbcdf42. 00000020:00000010:0.0:1784949279.158217:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000020:00000010:2.0:1784949279.158218:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000400:00000200:3.0:1784949279.158225:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000100:00000001:0.0:1784949279.158228:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784949279.158231:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000020:00000010:2.0:1784949279.158233:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000059e5b358. 00000020:00000010:0.0:1784949279.158236:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00000001:3.0:1784949279.158242:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1784949279.158249:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00100000:0.0:1784949279.158253:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009c295fd6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289686528:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949279.158255:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949279.158261:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:2.0:1784949279.158269:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008450ee2b. 00000100:00000200:0.0:1784949279.158273:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686528 02000000:00000001:2.0:1784949279.158280:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:0.0:1784949279.158282:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949279.158288:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949279.158291:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000200:3.0:1784949279.158297:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949279.158298:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949279.158300:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:0.0:1784949279.158311:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:2.0:1784949279.158315:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000000ef44eaa. 00000020:00000001:0.0:1784949279.158318:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000400:00000010:2.0:1784949279.158329:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000cd71eba3. 00000020:00000001:0.0:1784949279.158331:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949279.158334:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1784949279.158338:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949279.158341:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949279.158344:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949279.158348:0:337122:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:0.0:1784949279.158350:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:3.0:1784949279.158355:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:0.0:1784949279.158356:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.158361:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:0.0:1784949279.158363:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949279.158366:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.158372:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949279.158373:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949279.158378:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00001000:3.0:1784949279.158382:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=151 v=5 (5 5 5 5) 02000000:00000010:0.0:1784949279.158387:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000efbd7ee7. 00000100:00001000:3.0:1784949279.158391:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=151 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949279.158394:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949279.158402:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.158403:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949279.158410:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949279.158411:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949279.158418:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000040:0.0:1784949279.158420:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686528 00010000:00000001:3.0:1784949279.158425:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949279.158427:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949279.158436:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:0.0:1784949279.158442:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009c295fd6 x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:185/0 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949279.158450:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949279.158488:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949279.158489:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1784949279.158496:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.158498:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949279.158503:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:0.0:1784949279.158507:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=146 v=5 (1 1 1 1) 00000100:00000040:3.0:1784949279.158512:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949279.158517:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949279.158526:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949279.158535:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949279.158546:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949279.158550:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949279.158551:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004f010ef0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289686400:0@lo:400: 00000100:00000001:0.0:1784949279.158558:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949279.158562:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949279.158568:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949279.158571:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:0.0:1784949279.158575:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000629497f1. 00000100:00000040:3.0:1784949279.158581:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004f010ef0 x1871651289686400/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949279.158585:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289686528, offset 224 00000400:00000200:0.0:1784949279.158605:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949279.158612:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949279.158620:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:0.0:1784949279.158624:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000010:3.0:1784949279.158629:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 00000400:00000200:0.0:1784949279.158665:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3849:3849:236:4294967295] 192.168.202.137@tcp LPNI seq info [2457:2457:8:4294967295] 02000000:00000001:3.0:1784949279.158677:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949279.158685:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:0.0:1784949279.158688:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000040:3.0:1784949279.158691:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000800:00000200:0.0:1784949279.158702:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784949279.158705:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000800:00000010:0.0:1784949279.158710:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 02000000:00000010:3.0:1784949279.158714:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000004c284093. 00000800:00000200:0.0:1784949279.158725:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:3.0:1784949279.158729:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949279.158735:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:0.0:1784949279.158736:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000010:3.0:1784949279.158743:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004f010ef0. 00000800:00000200:0.0:1784949279.158745:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949279.158758:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949279.158762:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949279.158765:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949279.158771:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949279.158773:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949279.158780:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949279.158782:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949279.158787:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784949279.158790:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949279.158792:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949279.158795:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949279.158801:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949279.158806:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009c295fd6 x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:185/0 lens 224/224 e 0 to 0 dl 1784949290 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949279.158818:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009c295fd6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289686528:12345-192.168.202.137@tcp:400: Request processed in 609us (1050us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949279.158827:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 793 00000100:00000040:0.0:1784949279.158831:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949279.158834:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949279.158836:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949279.158840:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000734334de. 00000020:00000010:0.0:1784949279.158846:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002ffa19fa. 00000020:00000010:0.0:1784949279.158851:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000020:00000040:0.0:1784949279.158857:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949279.158860:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949279.159025:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949279.159032:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000400:00000200:2.0:1784949279.159040:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949279.159048:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949279.159049:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949279.159059:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000629497f1 00000800:00000001:0.0:1784949279.159065:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949279.159065:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000629497f1. 00000800:00000001:0.0:1784949279.159076:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949279.159077:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949279.159082:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949279.159084:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949279.159096:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949279.159110:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e200 00000400:00000200:0.0:1784949279.159118:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x94f9 [1] + 224 00000800:00000001:0.0:1784949279.159125:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949279.159135:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949279.159138:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949279.159142:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949279.159148:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949279.159152:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949279.159163:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949279.159187:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949279.159193:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949279.159196:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949279.159239:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949279.159244:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949279.159250:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949279.159252:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949279.159254:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005bb236bd. 00000400:00000200:2.0:1784949279.159261:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fcd1fd6e 00000400:00000010:2.0:1784949279.159263:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000fcd1fd6e. 00000100:00000001:2.0:1784949279.159266:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949279.159273:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949279.159287:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949279.159297:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949279.159299:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949279.159302:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949279.159304:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949279.159306:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949279.159308:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949279.159311:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949279.159318:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=146 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949279.159323:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=146 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949279.159326:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949279.159327:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949279.159330:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949279.159334:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949279.159343:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.159346:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949279.159349:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949279.159351:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949279.159353:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949279.159357:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949279.159365:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949279.159368:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949279.159369:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949279.159373:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949279.159381:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000009e01665e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289686528:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949279.159386:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949279.159389:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949279.159391:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000009e01665e x1871651289686528/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949279.159399:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949279.159401:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949279.159403:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 02000000:00000001:2.0:1784949279.159409:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949279.159411:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949279.159413:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949279.159415:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949279.159417:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000ffbbc320. 02000000:00000001:2.0:1784949279.159421:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949279.159423:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949279.159425:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000009e01665e. 00000100:00000001:2.0:1784949279.159429:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949279.159430:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949279.159432:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949279.159438:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949279.159439:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949279.159442:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949279.159444:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949279.199996:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306218539, at 4306218540 00000100:00000010:0.0:1784949279.200010:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:3.0:1784949279.200028:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306218539, at 4306218540 00000100:00000010:3.0:1784949279.200043:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000040:1.0:1784949279.200047:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306218539, at 4306218540 00000100:00000040:2.0:1784949279.200059:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306218539, at 4306218540 00000100:00000010:1.0:1784949279.200062:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:2.0:1784949279.200078:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000400:00000100:1.0:1784949279.668143:0:343953:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.8:988 00000400:00000100:0.0:1784949279.668151:0:343911:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.12:988 00000400:00000100:2.0:1784949279.668178:0:344109:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.14:988 00000400:00000100:3.0:1784949279.668181:0:344110:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.29:988 00000400:00000100:1.0:1784949279.668268:0:343953:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.8@tcp at host 111.222.173.8:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:0.0:1784949279.668275:0:343911:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.12@tcp at host 111.222.173.12:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784949279.668288:0:344110:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.29@tcp at host 111.222.173.29:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784949279.668293:0:344109:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.14@tcp at host 111.222.173.14:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949279.668296:0:343953:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.8@tcp: down 00000400:00000200:0.0:1784949279.668305:0:343911:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.12@tcp: down 00000400:00000200:1.0:1784949279.668313:0:343953:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784949279.668315:0:344110:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.29@tcp: down 00000400:00000200:2.0:1784949279.668322:0:344109:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.14@tcp: down 00000400:00000200:0.0:1784949279.668329:0:343911:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784949279.668331:0:343953:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.8@tcp 00000800:00000010:1.0:1784949279.668340:0:343953:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000bc928ba2. 00000800:00000100:0.0:1784949279.668344:0:343911:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.12@tcp 00000800:00000200:3.0:1784949279.668349:0:344110:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.173.29@tcp down 00000400:00000200:2.0:1784949279.668350:0:344109:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:0.0:1784949279.668352:0:343911:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009dcd7bb6. 00000400:00000200:1.0:1784949279.668354:0:343953:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949279.668367:0:343911:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949279.668370:0:343953:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949279.668379:0:343911:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949279.668386:0:343953:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11552 health :900 00000400:00000200:1.0:1784949279.668397:0:343953:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949279.668401:0:343911:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949279.668409:0:343953:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:0.0:1784949279.668414:0:343911:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:1.0:1784949279.668422:0:343953:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:0.0:1784949279.668446:0:343911:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000100:3.0:1784949279.668509:0:344015:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.10:988 00000400:00000100:3.0:1784949279.668578:0:344015:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.10@tcp at host 111.222.173.10:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784949279.668590:0:344015:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.10@tcp: down 00000400:00000200:3.0:1784949279.668597:0:344015:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:1.0:1784949279.668605:0:344010:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.6:988 00000400:00000100:0.0:1784949279.668659:0:344014:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.4:988 00000400:00000100:0.0:1784949279.668731:0:344014:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.4@tcp at host 111.222.173.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784949279.668732:0:344010:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.6@tcp at host 111.222.173.6:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949279.668747:0:344014:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.4@tcp: down 00000400:00000200:0.0:1784949279.668754:0:344014:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949279.668761:0:344010:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.6@tcp: down 00000800:00000100:0.0:1784949279.668769:0:344014:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.4@tcp 00000400:00000200:1.0:1784949279.668776:0:344010:0:(router.c:1888:lnet_notify()) Old news 00000800:00000010:0.0:1784949279.668777:0:344014:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000005c010cf6. 00000400:00000200:0.0:1784949279.668787:0:344014:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949279.668798:0:344014:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: NETWORK_TIMEOUT 00000800:00000100:1.0:1784949279.668800:0:344010:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.6@tcp 00000400:00000200:0.0:1784949279.668804:0:344014:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000800:00000010:1.0:1784949279.668810:0:344010:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008d5370ec. 00000400:00000200:0.0:1784949279.668814:0:344014:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00000400:00000200:0.0:1784949279.668827:0:344014:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000200:1.0:1784949279.668829:0:344010:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949279.668846:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949279.668848:0:344010:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949279.668864:0:344010:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949279.668865:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000100:2.0:1784949279.668878:0:343654:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.2:988 00000400:00000001:3.0:1784949279.668880:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1784949279.668883:0:344010:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000001:3.0:1784949279.668887:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949279.668897:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:3.0:1784949279.668902:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000400:00000200:1.0:1784949279.668967:0:344010:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:3.0:1784949279.668967:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949279.668989:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:3.0:1784949279.668999:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e120de1c. 00000400:00000200:3.0:1784949279.669009:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-110 00000400:00000100:2.0:1784949279.669018:0:343654:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.2@tcp at host 111.222.173.2:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784949279.669020:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -110 00000400:00000200:3.0:1784949279.669035:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000400:00000200:2.0:1784949279.669045:0:343654:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.2@tcp: down 00000400:00000200:3.0:1784949279.669053:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000200:2.0:1784949279.669054:0:343654:0:(router.c:1888:lnet_notify()) Old news 00000400:00000001:3.0:1784949279.669063:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949279.669069:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000100:2.0:1784949279.669074:0:343654:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.2@tcp 00000400:00000200:3.0:1784949279.669075:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000400:00000010:3.0:1784949279.669084:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000800:00000010:2.0:1784949279.669087:0:343654:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e706e181. 00000400:00000200:3.0:1784949279.669091:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949279.669097:0:343654:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949279.669100:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:3.0:1784949279.669110:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000075318cf7. 00000400:00000200:2.0:1784949279.669116:0:343654:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949279.669126:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-110 00000400:00000200:2.0:1784949279.669128:0:343654:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949279.669137:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -110 00000400:00000200:2.0:1784949279.669143:0:343654:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:3.0:1784949279.669150:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000400:00000200:2.0:1784949279.669163:0:343654:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:3.0:1784949279.669176:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:3.0:1784949279.669182:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949279.669184:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949279.669186:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000400:00000010:3.0:1784949279.669188:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000400:00000200:3.0:1784949279.669193:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949279.669198:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:3.0:1784949279.669202:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000e6d8b8ad. 00000400:00000200:3.0:1784949279.669209:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-110 00000400:00000200:3.0:1784949279.669213:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -110 00000400:00000200:3.0:1784949279.669217:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:3.0:1784949279.669220:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000400:00000001:3.0:1784949279.669223:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949279.669225:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949279.669227:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000010:3.0:1784949279.669230:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000400:00000200:3.0:1784949279.669235:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949279.669243:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:3.0:1784949279.669248:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000d531934e. 00000400:00000200:3.0:1784949279.669256:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-110 00000400:00000200:3.0:1784949279.669261:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -110 00000400:00000200:3.0:1784949279.669268:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:3.0:1784949279.669275:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:3.0:1784949279.669278:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949279.669279:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949279.669282:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:3.0:1784949279.669284:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:3.0:1784949279.669291:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949279.669297:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:3.0:1784949279.669300:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000b241612. 00000400:00000200:3.0:1784949279.669307:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-110 00000400:00000200:3.0:1784949279.669310:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -110 00000400:00000200:3.0:1784949279.669315:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00010000:00000001:2.0:1784949279.988029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949279.988039:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949279.988047:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949279.988049:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949279.988174:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949279.988181:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949279.988185:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949280.180052:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949280.180080:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949280.180083:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949280.180091:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949280.180095:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949280.180100:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949280.180104:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949280.372026:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:3.0:1784949280.372040:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000fe7e8b46. 00000400:00000010:3.0:1784949280.372054:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f7a7941e. 00000400:00000010:3.0:1784949280.372062:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003cb14b07. 00000400:00000010:3.0:1784949280.372073:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b4c93ba0. 00000400:00000200:3.0:1784949280.372080:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949280.372098:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949280.372106:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949280.372111:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3850:3850:241:4294967295] 192.168.202.137@tcp LPNI seq info [2458:2458:8:4294967295] 00000400:00000200:3.0:1784949280.372120:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:3.0:1784949280.372130:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:3.0:1784949280.372137:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949280.372143:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000800:00000200:3.0:1784949280.372151:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949280.372161:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949280.372167:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 2, nob 96 niov 1 nkiov 0 00000400:00000200:3.0:1784949280.372194:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11554 health :500 00000800:00000200:1.0:1784949280.372425:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949280.372434:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000400:00000200:1.0:1784949280.372446:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949280.372452:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949280.372460:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784949280.372468:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1784949280.372472:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949280.372475:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949280.372482:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949280.372492:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949280.372505:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949280.372524:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949280.372537:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949280.372549:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949280.372555:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3851:3851:241:4294967295] 192.168.202.137@tcp LPNI seq info [2459:2459:8:4294967295] 00000400:00000200:2.0:1784949280.372567:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949280.372573:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949280.372578:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000800:00000200:2.0:1784949280.372584:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949280.372590:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949280.372594:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949280.372704:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949280.372780:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949280.372786:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000400:00000200:2.0:1784949280.372794:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949280.372801:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949280.372803:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:1.0:1784949280.372816:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949280.372822:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949280.372830:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949280.372840:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000800:00000001:0.0:1784949280.372843:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949280.372861:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x950d 00000800:00000001:1.0:1784949280.372870:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949280.372882:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949280.372886:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949280.372890:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949280.372897:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003cb14b07 00000400:00000010:1.0:1784949280.372901:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003cb14b07. 00000400:00000200:1.0:1784949280.372942:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:1.0:1784949280.372946:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000fe7e8b46. 00000400:00000010:1.0:1784949280.372951:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f7a7941e. 00000800:00000001:1.0:1784949280.372962:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949280.372966:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949281.012034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949281.012044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949281.012052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949281.012056:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949281.012060:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949281.012063:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949281.012067:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949281.012070:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949281.012072:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949281.012075:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949281.012209:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949281.012216:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949281.012221:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949281.204016:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949281.204036:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949281.204040:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949281.204048:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949281.204053:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949281.204058:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949281.204063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949282.036027:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949282.036037:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949282.036045:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949282.036047:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949282.036175:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949282.036182:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949282.036187:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949282.228104:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949282.228121:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949282.228125:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949282.228132:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949282.228135:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949282.228141:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949282.228145:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949283.060047:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949283.060057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949283.060066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949283.060068:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949283.060182:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949283.060189:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949283.060194:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949283.252231:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949283.252253:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949283.252257:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949283.252269:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949283.252275:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949283.252281:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949283.252287:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949284.084050:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949284.084061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949284.084073:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949284.084076:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949284.084182:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949284.084190:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949284.084196:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784949284.276072:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949284.276089:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949284.276103:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008223e251. 00000020:00000040:1.0:1784949284.276114:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949284.276123:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949284.276130:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784949284.276134:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 02000000:00000001:1.0:1784949284.276139:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949284.276149:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:1.0:1784949284.276154:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a21d8ef3. 00000100:00000001:0.0:1784949284.276155:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949284.276164:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949284.276169:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949284.276171:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949284.276174:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949284.276181:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949284.276191:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949284.276229:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949284.276234:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949284.276245:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008223e251] to pc [ptlrpcd_00_01+1] req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949284.276294:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949284.276295:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.276304:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1784949284.276306:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:3.0:1784949284.276312:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1784949284.276319:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949284.276328:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1784949284.276329:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949284.276346:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949284.276349:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949284.276355:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949284.276359:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006a448376. 00000100:00000001:3.0:1784949284.276367:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1784949284.276369:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949284.276372:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949284.276380:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949284.276383:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949284.276386:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949284.276389:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949284.276393:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949284.276396:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949284.276403:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289686656:0@lo:400: 02000000:00000010:1.0:1784949284.276406:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000007a3f8609. 00000100:00000001:3.0:1784949284.276413:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949284.276418:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949284.276424:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949284.276429:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949284.276434:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1784949284.276435:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949284.276446:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dbce179e. 02000000:00000001:3.0:1784949284.276455:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949284.276466:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949284.276469:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949284.276471:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002fbcdf42. 00000100:00000040:1.0:1784949284.276480:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a448376] to pc [ptlrpcd_00_02+2] req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949284.276482:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 00000100:00000200:3.0:1784949284.276496:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686656, portal 25 00000100:00000001:3.0:1784949284.276506:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949284.276511:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637810048 : -115544071741568 : ffff96e9cca5df80) 00000100:00000040:3.0:1784949284.276523:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949284.276534:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949284.276540:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949284.276542:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949284.276551:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000100:00000040:1.0:1784949284.276555:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11561) 00000100:00000200:3.0:1784949284.276557:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289686656, offset 0 00000400:00000200:3.0:1784949284.276572:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949284.276584:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949284.276584:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949284.276622:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949284.276624:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949284.276643:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949284.276652:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949284.276666:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949284.276668:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e280 00000400:00000200:3.0:1784949284.276686:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8b09 [2] + 0 00000100:00000001:2.0:1784949284.276697:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:3.0:1784949284.276699:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000152ac2d7. 00000100:00000001:2.0:1784949284.276705:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949284.276709:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000004ba703f8 02000000:00000001:2.0:1784949284.276717:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784949284.276722:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784949284.276724:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949284.276734:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949284.276735:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00100000:2.0:1784949284.276749:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006a448376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289686784:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784949284.276750:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ba703f8 00000400:00000010:3.0:1784949284.276757:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000004ba703f8. 00000100:00000001:2.0:1784949284.276765:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949284.276768:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949284.276772:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:2.0:1784949284.276774:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784949284.276777:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f24717d4 x1871651289686656 msgsize 224 02000000:00000001:2.0:1784949284.276780:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949284.276791:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 02000000:00000001:2.0:1784949284.276792:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1784949284.276798:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 02000000:00000010:2.0:1784949284.276804:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000059e5b358. 02000000:00000001:2.0:1784949284.276811:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.276819:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:2.0:1784949284.276820:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008b17e867. 00000400:00000200:3.0:1784949284.276833:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000400:00000010:2.0:1784949284.276835:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000010:3.0:1784949284.276838:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000100:00000001:3.0:1784949284.276848:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1784949284.276852:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686784, portal 10 00000100:00000200:3.0:1784949284.276856:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949284.276863:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949284.276870:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637802880 : -115544071748736 : ffff96e9cca5c380) 00000100:00000001:3.0:1784949284.276879:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949284.276889:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784949284.276890:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949284.276892:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949284.276897:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686656 00000100:00000001:2.0:1784949284.276918:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949284.276936:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949284.276943:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949284.276949:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949284.276961:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949284.276962:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949284.276963:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949284.276969:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000001:3.0:1784949284.276973:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1784949284.276977:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686656 00000100:00000200:2.0:1784949284.276978:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289686784, offset 0 00000020:00000001:1.0:1784949284.276983:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949284.276986:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1784949284.276989:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949284.276995:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306223659 00000400:00000200:2.0:1784949284.276996:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949284.276998:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949284.277008:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:3.0:1784949284.277009:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000020:00000001:1.0:1784949284.277017:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:2.0:1784949284.277020:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949284.277024:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1784949284.277027:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:2.0:1784949284.277032:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949284.277039:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949284.277040:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306223659 00000400:00000200:2.0:1784949284.277042:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:1.0:1784949284.277051:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000010:3.0:1784949284.277053:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000400:00000200:2.0:1784949284.277055:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784949284.277062:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949284.277065:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000400:00000200:2.0:1784949284.277068:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:1.0:1784949284.277073:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:3.0:1784949284.277078:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306223659 00000400:00000200:2.0:1784949284.277083:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3852:3852:241:4294967295] 192.168.202.137@tcp LPNI seq info [2460:2460:8:4294967295] 00000020:00000010:1.0:1784949284.277086:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000100:00000010:3.0:1784949284.277088:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000001:3.0:1784949284.277126:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:1.0:1784949284.277129:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000dc288910. 00000100:00000001:3.0:1784949284.277140:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306223659 00000020:00000010:1.0:1784949284.277144:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000400:00000200:2.0:1784949284.277147:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000010:3.0:1784949284.277153:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:3.0:1784949284.277165:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000040:1.0:1784949284.277172:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000200:2.0:1784949284.277172:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949284.277178:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949284.277187:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949284.277187:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949284.277188:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000100:00000001:1.0:1784949284.277199:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949284.277199:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949284.277207:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949284.277209:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949284.277210:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949284.277224:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.277225:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949284.277231:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:2.0:1784949284.277235:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784949284.277240:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1784949284.277244:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949284.277245:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949284.277254:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949284.277255:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949284.277264:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.277272:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949284.277281:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2242 usecs (3 jiffies) 00000100:00000001:1.0:1784949284.277284:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949284.277287:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306223620 00000100:00000001:1.0:1784949284.277291:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949284.277293:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306223659 00000100:00000001:2.0:1784949284.277299:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306223620 00000100:00100000:1.0:1784949284.277307:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 818 00000100:00000001:2.0:1784949284.277308:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306223659 00000100:00000001:2.0:1784949284.277318:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306223620 00000100:00000040:1.0:1784949284.277320:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949284.277323:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306223659 00000100:00000001:1.0:1784949284.277330:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457130576 : -115544252421040 : ffff96e9c1e0ec50) 00000100:00000001:2.0:1784949284.277330:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306223620 00000100:00000001:2.0:1784949284.277341:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306223659 00000100:00000040:1.0:1784949284.277347:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f24717d4 x1871651289686656/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:190/0 lens 224/0 e 0 to 0 dl 1784949295 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949284.277348:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949284.277359:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.277370:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.277379:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949284.277383:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949284.277389:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1784949284.277392:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:2.0:1784949284.277395:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949284.277400:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949284.277403:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949284.277408:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1784949284.277410:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:2.0:1784949284.277413:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.277421:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949284.277429:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f24717d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289686656:12345-0@lo:400: 00000100:00000200:1.0:1784949284.277439:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686656 00000020:00000001:1.0:1784949284.277445:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949284.277454:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949284.277459:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949284.277463:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:1.0:1784949284.277465:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949284.277472:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000010:3.0:1784949284.277474:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000020:00000001:1.0:1784949284.277481:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:3.0:1784949284.277486:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784949284.277487:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949284.277496:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:3.0:1784949284.277500:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784949284.277503:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000200:3.0:1784949284.277508:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000020:00000001:1.0:1784949284.277511:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949284.277514:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000800:00000001:0.0:1784949284.277515:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949284.277518:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949284.277525:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949284.277527:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949284.277534:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000200:3.0:1784949284.277535:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949284.277536:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949284.277545:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009e9a21e3. 00000800:00000001:0.0:1784949284.277546:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949284.277555:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949284.277557:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949284.277570:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.277572:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949284.277579:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:1.0:1784949284.277582:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949284.277583:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1784949284.277635:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686656 00000400:00000200:0.0:1784949284.277644:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e300 00010000:00000001:1.0:1784949284.277649:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:3.0:1784949284.277650:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.277657:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949284.277662:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 178896 00010000:00000200:1.0:1784949284.277664:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f24717d4 x1871651289686656/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:190/0 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949284.277679:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:1.0:1784949284.277689:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949284.277694:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949284.277695:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949284.277705:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00001000:1.0:1784949284.277707:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=6 v=5 (1 1 1 1) 00000400:00000200:0.0:1784949284.277717:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949284.277721:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949284.277726:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:1.0:1784949284.277730:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000200:0.0:1784949284.277732:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:0.0:1784949284.277743:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e7ef9329. 00000100:00000001:1.0:1784949284.277743:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000040:0.0:1784949284.277752:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000e7ef9329 x1871651289686784 msgsize 224 02000000:00000001:1.0:1784949284.277754:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949284.277760:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949284.277769:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949284.277770:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949284.277779:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949284.277787:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000200:1.0:1784949284.277793:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289686656, offset 224 00000100:00000001:0.0:1784949284.277800:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784949284.277803:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949284.277818:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:0.0:1784949284.277821:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949284.277828:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949284.277835:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e280 00000400:00000200:1.0:1784949284.277849:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9511 [1] + 224 00000100:00000001:2.0:1784949284.277854:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784949284.277860:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:2.0:1784949284.277860:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686784 02000000:00000001:2.0:1784949284.277867:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949284.277871:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784949284.277873:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949284.277875:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949284.277883:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949284.277884:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949284.277887:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686784 00000020:00000001:2.0:1784949284.277892:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:1.0:1784949284.277894:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784949284.277896:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949284.277902:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1784949284.277918:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784949284.277943:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949284.277947:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949284.277969:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949284.277981:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949284.277990:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000084bc9b0a. 00000100:00000001:2.0:1784949284.277997:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949284.278000:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000084bc9b0a. 00000020:00000010:2.0:1784949284.278006:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b1cbf66f. 00000100:00000001:1.0:1784949284.278009:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784949284.278011:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085067f70. 00000020:00000010:2.0:1784949284.278019:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000200:1.0:1784949284.278024:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000100:00000040:2.0:1784949284.278029:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:1.0:1784949284.278031:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:2.0:1784949284.278036:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949284.278042:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949284.278043:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949284.278047:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949284.278049:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949284.278059:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949284.278061:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.278065:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949284.278070:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949284.278071:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784949284.278080:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949284.278084:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949284.278089:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784949284.278092:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949284.278100:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f24717d4 x1871651289686656/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:190/0 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949284.278113:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949284.278120:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949284.278120:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949284.278129:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f24717d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289686656:12345-0@lo:400: Request processed in 739us (1344us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949284.278130:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1784949284.278139:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 794 00000400:00000001:3.0:1784949284.278145:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784949284.278149:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000001:3.0:1784949284.278152:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1784949284.278157:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469558272 : -115544239993344 : ffff96e9c29e8e00) 00000400:00000010:3.0:1784949284.278159:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002fbcdf42. 00000100:00100000:1.0:1784949284.278165:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 818 00000100:00000040:2.0:1784949284.278175:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e7ef9329 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:190/0 lens 224/0 e 0 to 0 dl 1784949295 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949284.278176:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:3.0:1784949284.278176:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000400:00000010:3.0:1784949284.278184:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000100:00000001:1.0:1784949284.278187:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949284.278195:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784949284.278197:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784949284.278207:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:3.0:1784949284.278209:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949284.278210:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000dc288910. 00000020:00000010:2.0:1784949284.278213:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000084bc9b0a. 00000100:00000001:2.0:1784949284.278223:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949284.278225:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:2.0:1784949284.278233:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000084bc9b0a. 00000020:00000010:1.0:1784949284.278239:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000100:00100000:2.0:1784949284.278251:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e7ef9329 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289686784:12345-192.168.202.137@tcp:400: 00000020:00000040:1.0:1784949284.278254:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:3.0:1784949284.278259:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949284.278266:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686784 02000000:00000001:1.0:1784949284.278270:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:2.0:1784949284.278274:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000010:1.0:1784949284.278278:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009e9a21e3. 00000020:00000001:2.0:1784949284.278282:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949284.278290:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949284.278291:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949284.278293:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1784949284.278301:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949284.278302:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949284.278302:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949284.278307:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:3.0:1784949284.278316:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:2.0:1784949284.278317:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000200:1.0:1784949284.278319:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:2.0:1784949284.278326:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949284.278327:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:1.0:1784949284.278331:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000084bf7cfb. 00000100:00000001:3.0:1784949284.278333:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:2.0:1784949284.278335:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949284.278342:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949284.278343:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:1.0:1784949284.278347:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007b5c54ce. 00000020:00000001:2.0:1784949284.278349:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949284.278350:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949284.278360:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00001000:3.0:1784949284.278363:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=6 v=5 (5 5 5 5) 00000100:00000001:2.0:1784949284.278367:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949284.278375:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00001000:3.0:1784949284.278375:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=6 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949284.278385:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:2.0:1784949284.278386:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008450ee2b. 00000100:00000001:3.0:1784949284.278389:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949284.278392:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949284.278395:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:2.0:1784949284.278398:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949284.278400:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949284.278403:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.278406:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949284.278411:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686784 00000100:00000040:3.0:1784949284.278417:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949284.278418:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949284.278429:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e7ef9329 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:190/0 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949284.278454:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1784949284.278463:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949284.278464:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00010000:00000001:2.0:1784949284.278469:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.278470:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949284.278482:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=151 v=5 (1 1 1 1) 00000100:00000040:3.0:1784949284.278487:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949284.278494:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949284.278504:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949284.278514:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:3.0:1784949284.278517:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000008223e251 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289686656:0@lo:400: 02000000:00000001:2.0:1784949284.278519:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949284.278524:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.278535:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949284.278536:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949284.278545:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949284.278546:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:2.0:1784949284.278553:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000100:00000040:3.0:1784949284.278555:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000008223e251 x1871651289686656/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949284.278564:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289686784, offset 224 00000100:00000001:3.0:1784949284.278579:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:2.0:1784949284.278583:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949284.278586:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949284.278591:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dbce179e. 00000400:00000200:2.0:1784949284.278643:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:3.0:1784949284.278646:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949284.278652:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949284.278656:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:2.0:1784949284.278661:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3853:3853:241:4294967295] 192.168.202.137@tcp LPNI seq info [2461:2461:8:4294967295] 00000020:00000001:3.0:1784949284.278662:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949284.278668:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a21d8ef3. 02000000:00000001:3.0:1784949284.278675:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949284.278679:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949284.278682:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000008223e251. 00000100:00000001:3.0:1784949284.278689:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949284.278694:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949284.278702:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949284.278703:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949284.278709:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949284.278713:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949284.278720:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1784949284.278721:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949284.278724:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949284.278734:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000800:00000200:2.0:1784949284.278746:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949284.278756:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949284.278761:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949284.278778:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949284.278782:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949284.278785:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949284.278787:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949284.278789:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949284.278795:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e7ef9329 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:190/0 lens 224/224 e 0 to 0 dl 1784949295 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949284.278806:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e7ef9329 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289686784:12345-192.168.202.137@tcp:400: Request processed in 600us (1053us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949284.278816:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 794 00000100:00000040:2.0:1784949284.278820:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949284.278824:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949284.278827:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949284.278831:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085067f70. 00000020:00000010:2.0:1784949284.278837:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949284.278842:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b1cbf66f. 00000020:00000040:2.0:1784949284.278847:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949284.278850:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949284.278987:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949284.278994:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000400:00000200:1.0:1784949284.279003:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949284.279004:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949284.279014:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949284.279017:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949284.279023:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000800:00000001:3.0:1784949284.279024:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949284.279030:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949284.279031:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000100:00000001:1.0:1784949284.279040:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:3.0:1784949284.279046:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949284.279047:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949284.279064:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e300 00000400:00000200:3.0:1784949284.279073:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9519 [1] + 224 00000800:00000001:3.0:1784949284.279081:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949284.279093:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949284.279096:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949284.279100:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949284.279105:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949284.279109:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949284.279120:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949284.279152:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949284.279159:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949284.279162:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949284.279199:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949284.279204:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949284.279209:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949284.279211:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949284.279213:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008b17e867. 00000400:00000200:2.0:1784949284.279237:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784949284.279239:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:2.0:1784949284.279242:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949284.279249:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949284.279260:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949284.279270:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949284.279271:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949284.279274:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949284.279276:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949284.279278:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949284.279280:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949284.279282:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949284.279288:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=151 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949284.279292:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=151 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949284.279295:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949284.279297:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949284.279299:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949284.279303:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949284.279312:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.279316:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949284.279318:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949284.279321:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949284.279322:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949284.279326:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949284.279333:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949284.279336:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949284.279337:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949284.279341:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949284.279349:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006a448376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289686784:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949284.279354:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949284.279356:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949284.279359:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006a448376 x1871651289686784/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949284.279367:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949284.279369:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949284.279371:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000059e5b358. 02000000:00000001:2.0:1784949284.279374:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949284.279375:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949284.279377:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949284.279379:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949284.279381:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000007a3f8609. 02000000:00000001:2.0:1784949284.279384:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949284.279386:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949284.279387:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006a448376. 00000100:00000001:2.0:1784949284.279391:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949284.279392:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949284.279394:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949284.279399:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949284.279401:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949284.279403:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949284.279406:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949284.320042:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306223659, at 4306223660 00000100:00000040:3.0:1784949284.320046:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306223659, at 4306223660 00000100:00000040:1.0:1784949284.320056:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306223659, at 4306223660 00000100:00000010:0.0:1784949284.320059:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000040:2.0:1784949284.320059:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306223659, at 4306223660 00000100:00000010:3.0:1784949284.320062:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000010:1.0:1784949284.320076:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000010:2.0:1784949284.320080:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00010000:00000001:2.0:1784949285.108044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949285.108054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949285.108062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949285.108064:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949285.108196:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949285.108203:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949285.108207:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949285.300043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949285.300060:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949285.300063:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949285.300069:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949285.300074:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949285.300079:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949285.300083:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949286.132036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949286.132045:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949286.132053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949286.132055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949286.132155:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949286.132164:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949286.132169:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949286.324026:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949286.324044:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949286.324046:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949286.324052:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949286.324055:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949286.324059:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949286.324063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949287.155984:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949287.155991:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949287.155996:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949287.155997:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949287.156032:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949287.156036:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949287.156039:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949287.348025:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949287.348042:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949287.348045:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949287.348052:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949287.348055:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949287.348060:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949287.348063:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949288.180018:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949288.180026:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949288.180033:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949288.180034:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949288.180144:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949288.180150:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949288.180153:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949288.371988:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949288.371998:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949288.372013:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949263/real 1784949263] req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949288 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1784949288.372037:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949288.372040:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949288.372043:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000035db9be3. 00000400:00000200:0.0:1784949288.372051:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:0.0:1784949288.372054:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:0.0:1784949288.372062:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949288.372066:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949288 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949288.372078:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949288 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949288.372087:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949288.372089:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.372092:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949288.372094:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949288.372097:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949288 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949288.372131:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.372141:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949288.372142:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949288.372146:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949288.372149:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949288.372152:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949288.372156:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949288 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949288.372165:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949288.372170:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.372172:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949288.372173:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.372176:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949288.372179:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949288.372183:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949288.372213:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:0.0:1784949288.372217:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949288.372221:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949288 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949288.372231:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289685632:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784949288.372239:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.372243:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949288.372247:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ca646268 x1871651289685632/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949288 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949288.372256:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949288.372260:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949288.372265:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000079de2679. 02000000:00000001:0.0:1784949288.372275:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:1.0:1784949288.372276:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1784949288.372278:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949288.372284:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:1.0:1784949288.372289:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000020:00000001:0.0:1784949288.372291:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949288.372297:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000003e4182b6. 02000000:00000001:0.0:1784949288.372301:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000010:1.0:1784949288.372303:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005126bdfa. 02000000:00000001:0.0:1784949288.372305:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949288.372309:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ca646268. 00000020:00000040:1.0:1784949288.372313:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949288.372317:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949288.372321:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949288.372323:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949288.372325:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949288.372328:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784949288.372331:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949288.372335:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949288.372337:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949288.372338:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949288.372344:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949288.372352:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000009e7fe55b. 00000100:00000001:1.0:1784949288.372364:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.372373:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949288.372390:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949288.372393:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.372399:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005126bdfa] to pc [ptlrpcd_00_03+3] req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949288.372435:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949288.372438:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949288.372442:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1784949288.372445:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1784949288.372447:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1784949288.372452:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1784949288.372457:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11535 00000400:00000200:1.0:1784949288.372468:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00080000:1.0:1784949288.372474:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11534 00000400:00000200:1.0:1784949288.372480:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:3.0:1784949288.372486:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949288.372487:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949288.372496:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949288.372496:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1784949288.372502:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784949288.372503:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949288.372510:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000040:3.0:1784949288.372516:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949288.372519:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949288.372523:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:1.0:1784949288.372531:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949288.372537:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000001:3.0:1784949288.372539:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1784949288.372547:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:3.0:1784949288.372548:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.372550:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949288.372583:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784949288.372590:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1784949288.372591:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949288.372596:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949288.372597:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949288.372598:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784949288.372608:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.372609:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000100:00000001:1.0:1784949288.372619:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00100000:3.0:1784949288.372620:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289686912:0@lo:400: 00000020:00000040:1.0:1784949288.372628:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:3.0:1784949288.372631:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1784949288.372639:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000001:3.0:1784949288.372643:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1784949288.372645:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 02000000:00000001:3.0:1784949288.372651:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949288.372652:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949288.372662:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1784949288.372664:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1784949288.372670:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949288.372676:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949288.372677:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 00000100:00000001:1.0:1784949288.372679:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1784949288.372685:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949288.372688:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949288.372694:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cd114eaa. 00000020:00000040:1.0:1784949288.372700:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000010:3.0:1784949288.372703:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002fbcdf42. 02000000:00000001:1.0:1784949288.372704:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1784949288.372709:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949288.372716:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949288.372719:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949288.372722:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949288.372722:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000142456d0. 02000000:00000010:1.0:1784949288.372733:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000068d8ca1f. 00000100:00000200:3.0:1784949288.372740:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289686912, portal 25 00000100:00000001:1.0:1784949288.372741:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949288.372753:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1784949288.372756:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949288.372759:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637815424 : -115544071736192 : ffff96e9cca5f480) 00000100:00000001:1.0:1784949288.372780:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1784949288.372784:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949288.372785:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.372797:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd114eaa] to pc [ptlrpcd_rcv+-1] req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949288.372851:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949288.372857:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949288.372862:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000100:00000001:1.0:1784949288.372863:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949288.372867:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289686912, offset 0 00000100:00000040:1.0:1784949288.372869:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949288.372877:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:3.0:1784949288.372878:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000010:1.0:1784949288.372883:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000029a32fda. 00000020:00000040:1.0:1784949288.372888:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949288.372891:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949288.372893:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949288.372895:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949288.372901:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784949288.372917:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e380 02000000:00000010:1.0:1784949288.372932:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000076916763. 00000100:00000001:1.0:1784949288.372939:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.372946:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949288.372956:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949288.372956:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949288.372961:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949288.372964:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949288.372969:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949288.372970:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029a32fda] to pc [ptlrpcd_00_00+0] req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949288.372984:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949288.372984:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8b25 [2] + 0 00000400:00000010:3.0:1784949288.372999:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000ac0969b3. 00000400:00000200:3.0:1784949288.373010:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000aad879a6 00000100:00000001:0.0:1784949288.373012:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949288.373012:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949288.373017:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.373022:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11565) 02000000:00000001:0.0:1784949288.373023:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1784949288.373024:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:0.0:1784949288.373030:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949288.373038:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949288.373039:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949288.373040:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949288.373046:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1784949288.373052:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289687040:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949288.373053:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1784949288.373054:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aad879a6 00000100:00000001:1.0:1784949288.373062:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1784949288.373063:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000aad879a6. 00000100:00000001:0.0:1784949288.373064:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1784949288.373071:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949288.373074:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949288.373074:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949288.373075:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949288.373082:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784949288.373085:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949288.373092:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000028574ea9 x1871651289686912 msgsize 224 00000100:00000001:0.0:1784949288.373094:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949288.373103:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.373126:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1784949288.373130:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:0.0:1784949288.373132:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949288.373133:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949288.373136:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949288.373139:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949288.373143:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949288.373145:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949288.373149:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949288.373150:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949288.373155:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949288.373157:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949288.373160:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949288.373161:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949288.373168:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289687168:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949288.373176:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784949288.373178:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 02000000:00000001:2.0:1784949288.373181:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:3.0:1784949288.373184:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 02000000:00000001:2.0:1784949288.373188:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949288.373194:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949288.373195:0:337122:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:2.0:1784949288.373209:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000059e5b358. 00000100:00000200:3.0:1784949288.373213:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949288.373215:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949288.373243:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008b17e867. 00000100:00000001:1.0:1784949288.373244:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949288.373252:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289686912 00000400:00000010:2.0:1784949288.373255:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 02000000:00000001:1.0:1784949288.373258:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949288.373264:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784949288.373267:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289687168, portal 10 00000100:00000001:3.0:1784949288.373267:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949288.373269:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.373277:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1784949288.373279:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949288.373280:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.373281:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637809152 : -115544071742464 : ffff96e9cca5dc00) 00000100:00100000:1.0:1784949288.373291:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289686912 00000020:00000001:1.0:1784949288.373298:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:2.0:1784949288.373299:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949288.373303:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949288.373310:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949288.373314:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949288.373321:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949288.373327:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949288.373329:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949288.373336:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949288.373340:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949288.373343:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:1.0:1784949288.373348:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:3.0:1784949288.373349:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949288.373350:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949288.373358:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949288.373362:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000100:00000001:3.0:1784949288.373362:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306227755 00000020:00000010:1.0:1784949288.373363:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000200:2.0:1784949288.373373:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289687168, offset 0 00000100:00000010:3.0:1784949288.373377:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000020:00000010:1.0:1784949288.373379:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000400:00000200:2.0:1784949288.373388:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949288.373391:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:1.0:1784949288.373392:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000319b9e18. 00000400:00000200:2.0:1784949288.373401:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000010:1.0:1784949288.373403:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000400:00000200:2.0:1784949288.373409:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1784949288.373415:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000200:2.0:1784949288.373418:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:3.0:1784949288.373419:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306227755 00000100:00000001:1.0:1784949288.373421:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949288.373427:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:2.0:1784949288.373430:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000010:3.0:1784949288.373431:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000001:1.0:1784949288.373432:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949288.373443:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949288.373443:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949288.373449:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949288.373452:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306227755 00000400:00000200:2.0:1784949288.373458:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3854:3854:241:4294967295] 192.168.202.137@tcp LPNI seq info [2462:2462:8:4294967295] 00000100:00000010:3.0:1784949288.373461:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00000001:3.0:1784949288.373469:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949288.373470:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949288.373476:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306227755 00000100:00000010:3.0:1784949288.373485:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:1.0:1784949288.373491:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784949288.373491:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949288.373493:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949288.373499:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949288.373504:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949288.373510:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949288.373512:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 819 00000800:00000200:2.0:1784949288.373516:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949288.373518:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949288.373519:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.373523:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949288.373525:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000010:2.0:1784949288.373528:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c2b2d4f6. 00000100:00000001:3.0:1784949288.373528:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949288.373529:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457133648 : -115544252417968 : ffff96e9c1e0f850) 00000100:00000001:0.0:1784949288.373531:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949288.373540:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949288.373542:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949288.373547:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000028574ea9 x1871651289686912/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:194/0 lens 224/0 e 0 to 0 dl 1784949299 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949288.373550:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000800:00000200:2.0:1784949288.373551:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949288.373581:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949288.373589:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949288.373591:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949288.373594:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949288.373616:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000800:00000200:2.0:1784949288.373620:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784949288.373620:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949288.373623:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949288.373633:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000800:00000200:2.0:1784949288.373634:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c2b2d4f6 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:1.0:1784949288.373643:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000028574ea9 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289686912:12345-0@lo:400: 00000100:00000200:1.0:1784949288.373650:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289686912 00000020:00000001:1.0:1784949288.373656:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949288.373661:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:2.0:1784949288.373662:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949288.373665:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949288.373673:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:2.0:1784949288.373673:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2243 usecs (3 jiffies) 00000020:00000001:1.0:1784949288.373676:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:2.0:1784949288.373683:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306227716 00000020:00000001:1.0:1784949288.373685:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949288.373691:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.373693:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306227755 00000020:00000001:1.0:1784949288.373697:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949288.373701:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306227716 00000020:00000001:1.0:1784949288.373703:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949288.373708:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.373710:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306227755 00000020:00000001:1.0:1784949288.373713:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949288.373719:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306227716 00000100:00000001:1.0:1784949288.373721:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784949288.373726:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306227755 02000000:00000001:1.0:1784949288.373728:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949288.373735:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306227716 02000000:00000010:1.0:1784949288.373737:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000a41d8ff9. 02000000:00000001:1.0:1784949288.373743:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.373743:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306227755 00000100:00000001:1.0:1784949288.373750:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.373755:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949288.373756:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949288.373767:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289686912 00000100:00000001:2.0:1784949288.373768:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949288.373774:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:2.0:1784949288.373779:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:1.0:1784949288.373783:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000028574ea9 x1871651289686912/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:194/0 lens 224/224 e 0 to 0 dl 1784949299 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949288.373789:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949288.373802:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949288.373805:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1784949288.373807:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.373813:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00001000:1.0:1784949288.373815:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=10 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949288.373842:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1784949288.373842:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:0.0:1784949288.373845:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:1.0:1784949288.373847:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949288.373850:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000010:0.0:1784949288.373856:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c2b2d4f6. 00000100:00000001:1.0:1784949288.373856:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:2.0:1784949288.373858:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949288.373867:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000200:0.0:1784949288.373869:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949288.373869:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949288.373871:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949288.373873:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949288.373884:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949288.373889:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949288.373892:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949288.373895:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000001:3.0:1784949288.373902:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949288.373903:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000400:00000010:1.0:1784949288.373905:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000010:0.0:1784949288.373908:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000100:00000001:0.0:1784949288.373916:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:3.0:1784949288.373937:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949288.373948:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000200:1.0:1784949288.373951:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289686912, offset 224 00000100:00000200:0.0:1784949288.373960:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949288.373963:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e480 00000400:00000200:1.0:1784949288.373968:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949288.373986:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 179120 00000100:00000001:0.0:1784949288.373987:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784949288.373989:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:0.0:1784949288.373999:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949288.374005:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949288.374012:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e380 00000800:00000001:3.0:1784949288.374018:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.374019:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949288.374029:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x952d [1] + 224 00000400:00000200:3.0:1784949288.374030:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949288.374031:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784949288.374044:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949288.374045:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949288.374058:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949288.374060:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949288.374063:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784949288.374066:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:3.0:1784949288.374070:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046cb422a. 00000100:00000200:1.0:1784949288.374075:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949288.374077:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000046cb422a x1871651289687168 msgsize 224 00000100:00100000:3.0:1784949288.374095:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:1.0:1784949288.374125:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949288.374137:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:3.0:1784949288.374149:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949288.374155:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949288.374156:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949288.374173:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784949288.374179:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:1.0:1784949288.374185:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949288.374186:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949288.374188:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00100000:2.0:1784949288.374193:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289687168 02000000:00000001:2.0:1784949288.374198:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949288.374200:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.374202:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949288.374208:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949288.374209:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 02000000:00000001:2.0:1784949288.374216:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949288.374217:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949288.374225:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949288.374225:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949288.374226:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289687168 00000100:00000001:0.0:1784949288.374235:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949288.374236:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000001:1.0:1784949288.374237:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000040:2.0:1784949288.374244:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000001:0.0:1784949288.374247:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1784949288.374250:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000028574ea9 x1871651289686912/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:194/0 lens 224/224 e 0 to 0 dl 1784949299 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949288.374253:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000001:0.0:1784949288.374255:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:2.0:1784949288.374262:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000010:0.0:1784949288.374263:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002fbcdf42. 00000020:00000001:2.0:1784949288.374273:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949288.374282:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784949288.374286:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000142456d0 00000100:00100000:1.0:1784949288.374288:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000028574ea9 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289686912:12345-0@lo:400: Request processed in 691us (1189us total) trans 0 rc 0/0 00000400:00000010:0.0:1784949288.374294:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000142456d0. 00000100:00000001:2.0:1784949288.374294:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949288.374304:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000008aed2f1. 00000100:00000001:0.0:1784949288.374306:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784949288.374313:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 819 00000100:00000001:2.0:1784949288.374315:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1784949288.374321:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949288.374321:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000010:2.0:1784949288.374323:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000008aed2f1. 00000100:00000001:1.0:1784949288.374332:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000010:2.0:1784949288.374338:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b1cbf66f. 00000100:00000001:1.0:1784949288.374340:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949288.374351:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000319b9e18. 00000020:00000010:2.0:1784949288.374352:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085067f70. 00000100:00000200:0.0:1784949288.374353:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949288.374364:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000020:00000010:1.0:1784949288.374365:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949288.374371:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949288.374378:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949288.374378:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000100:00000001:0.0:1784949288.374383:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:2.0:1784949288.374384:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000040:1.0:1784949288.374386:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:0.0:1784949288.374390:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949288.374392:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949288.374396:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:1.0:1784949288.374398:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784949288.374398:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949288.374400:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949288.374408:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000a41d8ff9. 02000000:00000001:0.0:1784949288.374409:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.374409:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949288.374422:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:0.0:1784949288.374424:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=10 v=5 (5 5 5 5) 00000100:00000001:2.0:1784949288.374426:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949288.374427:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.374434:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00001000:0.0:1784949288.374439:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=10 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949288.374440:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949288.374441:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784949288.374453:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:1.0:1784949288.374454:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dbb46726. 00000100:00000001:0.0:1784949288.374456:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949288.374465:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:1.0:1784949288.374466:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f486321e. 00010000:00000001:0.0:1784949288.374468:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.374471:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.374478:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949288.374487:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949288.374492:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949288.374500:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949288.374501:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949288.374508:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1784949288.374515:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 795 00000100:00000001:0.0:1784949288.374516:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949288.374527:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000040:0.0:1784949288.374531:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949288.374534:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554953472 : -115544154598144 : ffff96e9c7b59500) 00000100:00000040:2.0:1784949288.374552:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000046cb422a x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:194/0 lens 224/0 e 0 to 0 dl 1784949299 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949288.374556:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289686912:0@lo:400: 00000100:00000001:0.0:1784949288.374586:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.374609:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949288.374614:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005126bdfa x1871651289686912/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949288.374639:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949288.374641:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949288.374645:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:2.0:1784949288.374647:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000008aed2f1. 02000000:00000010:0.0:1784949288.374648:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 00000100:00000001:2.0:1784949288.374655:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949288.374679:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:2.0:1784949288.374679:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000008aed2f1. 00000020:00000001:0.0:1784949288.374685:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949288.374689:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00100000:2.0:1784949288.374696:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000046cb422a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289687168:12345-192.168.202.137@tcp:400: 00000020:00000001:0.0:1784949288.374698:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949288.374704:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000009e7fe55b. 00000100:00000200:2.0:1784949288.374712:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289687168 02000000:00000001:0.0:1784949288.374715:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1784949288.374718:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:0.0:1784949288.374719:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:2.0:1784949288.374724:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000010:0.0:1784949288.374725:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005126bdfa. 00000020:00000001:2.0:1784949288.374731:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949288.374733:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:2.0:1784949288.374738:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949288.374739:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949288.374747:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:0.0:1784949288.374748:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.374757:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1784949288.374760:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949288.374763:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949288.374768:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949288.374769:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949288.374774:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949288.374779:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949288.374785:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949288.374789:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949288.374796:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949288.374802:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949288.374806:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949288.374812:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006e44748a. 02000000:00000001:2.0:1784949288.374817:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.374821:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949288.374825:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949288.374855:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289687168 00010000:00000001:2.0:1784949288.374859:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949288.374867:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000046cb422a x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:194/0 lens 224/224 e 0 to 0 dl 1784949299 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949288.374877:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949288.374879:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949288.374882:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=5 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949288.374886:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949288.374889:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949288.374891:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949288.374894:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949288.374895:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.374898:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949288.374900:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949288.374903:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000100:00000200:2.0:1784949288.374907:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289687168, offset 224 00000400:00000200:2.0:1784949288.374912:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949288.374920:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949288.374926:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3855:3855:241:4294967295] 192.168.202.137@tcp LPNI seq info [2463:2463:8:4294967295] 00000400:00000200:2.0:1784949288.374936:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949288.374941:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949288.374945:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000031deb17e. 00000800:00000200:2.0:1784949288.374949:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949288.374955:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949288.374958:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000031deb17e type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949288.375002:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949288.375006:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949288.375008:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949288.375010:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949288.375012:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949288.375017:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000046cb422a x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:194/0 lens 224/224 e 0 to 0 dl 1784949299 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949288.375028:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000046cb422a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289687168:12345-192.168.202.137@tcp:400: Request processed in 388us (949us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949288.375038:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 795 00000100:00000040:2.0:1784949288.375041:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949288.375043:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949288.375045:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949288.375049:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085067f70. 00000020:00000010:2.0:1784949288.375054:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949288.375058:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b1cbf66f. 00000020:00000040:2.0:1784949288.375062:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949288.375066:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949288.375126:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949288.375132:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000031deb17e. 00000400:00000200:3.0:1784949288.375153:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949288.375160:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949288.375165:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:3.0:1784949288.375167:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000100:00000001:3.0:1784949288.375171:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949288.375173:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949288.375178:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949288.375184:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949288.375187:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949288.375190:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949288.375198:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949288.375213:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e480 00000400:00000200:3.0:1784949288.375224:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9535 [1] + 224 00000800:00000001:3.0:1784949288.375233:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949288.375251:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949288.375255:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949288.375263:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949288.375269:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949288.375277:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949288.375296:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949288.375325:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949288.375333:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949288.375338:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.375388:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949288.375392:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949288.375397:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949288.375399:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949288.375401:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008b17e867. 00000400:00000200:2.0:1784949288.375406:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784949288.375408:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:2.0:1784949288.375410:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949288.375416:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949288.375425:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949288.375433:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949288.375435:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.375438:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949288.375440:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949288.375442:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949288.375443:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949288.375445:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949288.375451:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=5 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949288.375454:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=5 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949288.375456:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949288.375458:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949288.375460:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949288.375463:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949288.375471:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.375473:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949288.375475:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949288.375477:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949288.375478:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949288.375481:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949288.375487:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949288.375490:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949288.375491:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949288.375494:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949288.375502:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289687168:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949288.375506:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.375508:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949288.375510:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000029a32fda x1871651289687168/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949304 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949288.375517:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949288.375518:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949288.375520:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000059e5b358. 02000000:00000001:2.0:1784949288.375524:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949288.375525:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949288.375527:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949288.375529:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949288.375531:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000076916763. 02000000:00000001:2.0:1784949288.375533:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949288.375535:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949288.375536:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000029a32fda. 00000100:00000001:2.0:1784949288.375539:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949288.375540:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.375542:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949288.375546:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949288.375547:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949288.375549:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949288.375551:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949288.416001:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306227755, at 4306227756 00000100:00000040:1.0:1784949288.416003:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306227755, at 4306227756 00000100:00000040:3.0:1784949288.416012:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306227755, at 4306227756 00000100:00000010:0.0:1784949288.416015:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:2.0:1784949288.416018:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306227755, at 4306227756 00000100:00000010:1.0:1784949288.416019:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000010:3.0:1784949288.416028:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:2.0:1784949288.416033:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00010000:00000001:2.0:1784949289.204046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949289.204056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949289.204065:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949289.204067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949289.204176:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949289.204186:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949289.204193:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949289.396004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949289.396014:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949289.396042:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949288/real 0] req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949289 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.396062:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.396067:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949289.396069:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949289.396075:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949289 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.396085:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.396096:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949289.396098:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.396102:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949289.396107:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949289.396111:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949289.396119:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949289 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.396130:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949289.396137:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.396141:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949289.396142:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.396146:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949289.396150:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949289.396154:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949289.396172:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784949289.396177:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949289.396181:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949289 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949289.396193:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289687040:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949289.396204:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.396208:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949289.396214:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000cd114eaa x1871651289687040/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949289 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.396224:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784949289.396229:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949289.396232:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784949289.396236:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949289.396242:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 0000000068d8ca1f. 02000000:00000001:0.0:1784949289.396250:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949289.396253:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949289.396255:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000cd114eaa. 00000100:00000001:0.0:1784949289.396261:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949289.396264:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.396267:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.396272:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949289.396274:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.396278:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949289.396280:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949289.396319:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949289.396326:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949289.396333:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ca646268. 00000020:00000040:0.0:1784949289.396338:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949289.396344:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949289.396348:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949289.396354:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949289.396363:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000057c7ab4. 00000100:00000001:0.0:1784949289.396383:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949289.396391:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949289.396415:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949289.396420:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949289.396430:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca646268] to pc [ptlrpcd_00_01+1] req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.396473:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.396476:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784949289.396478:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:1.0:1784949289.396486:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1784949289.396487:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784949289.396489:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949289.396494:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949289.396498:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1784949289.396500:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949289.396504:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1784949289.396507:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.396518:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949289.396528:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11535 00000100:00000001:1.0:1784949289.396534:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949289.396540:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949289.396546:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 02000000:00000001:1.0:1784949289.396547:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1784949289.396582:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11560 02000000:00000001:1.0:1784949289.396585:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949289.396591:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949289.396592:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00100000:1.0:1784949289.396608:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289687296:0@lo:400: 00000100:00080000:0.0:1784949289.396609:0:89766:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1784949289.396621:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949289.396622:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949289.396629:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949289.396631:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784949289.396636:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949289.396638:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.396645:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949289.396647:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1784949289.396654:0:89766:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000010:1.0:1784949289.396657:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000229221cc. 00000020:00000040:0.0:1784949289.396658:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784949289.396666:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1784949289.396667:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949289.396674:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:1.0:1784949289.396678:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000aba79327. 00000020:00000001:0.0:1784949289.396681:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784949289.396687:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:1.0:1784949289.396692:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000040:0.0:1784949289.396693:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949289.396702:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949289.396706:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000200:1.0:1784949289.396708:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289687296, portal 25 00000100:00000001:0.0:1784949289.396712:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:1.0:1784949289.396714:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1784949289.396721:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000001:1.0:1784949289.396721:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469564544 : -115544239987072 : ffff96e9c29ea680) 00000100:00080000:0.0:1784949289.396732:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000040:1.0:1784949289.396736:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784949289.396744:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:0.0:1784949289.396753:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949289.396761:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1784949289.396766:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949289.396772:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949289.396773:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784949289.396783:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949289.396784:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000100:00000001:0.0:1784949289.396794:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:1.0:1784949289.396795:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289687296, offset 0 00000100:00000001:0.0:1784949289.396801:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949289.396807:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000010:0.0:1784949289.396811:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009d6fa0dc. 00000020:00000040:0.0:1784949289.396816:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784949289.396823:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949289.396826:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.396829:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949289.396832:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949289.396834:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949289.396842:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:0.0:1784949289.396847:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000000b08bf97. 00000100:00000001:0.0:1784949289.396854:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949289.396864:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e500 00000100:00100000:0.0:1784949289.396865:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784949289.396884:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8b41 [2] + 0 00000100:00000001:0.0:1784949289.396887:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949289.396892:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949289.396895:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b758fcf6. 00000100:00000040:0.0:1784949289.396902:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009d6fa0dc] to pc [ptlrpcd_rcv+-1] req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949289.396948:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000e65b0b8e 00000400:00000200:1.0:1784949289.396962:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949289.396970:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949289.396975:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949289.396978:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e65b0b8e 00000100:00000040:0.0:1784949289.396983:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1784949289.396984:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000e65b0b8e. 00000100:00000001:0.0:1784949289.396995:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949289.396998:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000010:0.0:1784949289.397002:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c69adecd. 00000100:00000200:1.0:1784949289.397003:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000020:00000040:0.0:1784949289.397011:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000040:1.0:1784949289.397014:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000009b82ab0d x1871651289687296 msgsize 224 00000100:00000001:0.0:1784949289.397017:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949289.397023:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949289.397026:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949289.397031:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949289.397033:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003a8a98bc. 00000100:00100000:1.0:1784949289.397033:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:2.0:1784949289.397038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949289.397050:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949289.397056:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949289.397056:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:2.0:1784949289.397065:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949289.397076:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949289.397077:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784949289.397092:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000400:00000010:1.0:1784949289.397099:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000100:00000001:2.0:1784949289.397109:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949289.397112:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949289.397118:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949289.397119:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.397125:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949289.397126:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949289.397131:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949289.397139:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949289.397145:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c69adecd] to pc [ptlrpcd_00_02+2] req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949289.397150:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.397164:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1784949289.397166:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000009d6fa0dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289687424:192.168.202.137@tcp:8: 00000100:00000040:1.0:1784949289.397175:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949289.397184:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949289.397192:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949289.397199:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949289.397208:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949289.397211:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.397216:0:337120:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:2.0:1784949289.397217:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000077ad1613. 00000100:00000001:0.0:1784949289.397223:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949289.397224:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.397227:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949289.397239:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11566) 00000100:00000001:1.0:1784949289.397240:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306228779 00000400:00000010:2.0:1784949289.397241:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000008b17e867. 00000100:00000010:1.0:1784949289.397255:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001a2e8205. 00000400:00000010:2.0:1784949289.397259:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000001:1.0:1784949289.397269:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000200:2.0:1784949289.397270:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289687424, portal 4 00000100:00000001:2.0:1784949289.397279:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949289.397282:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469560960 : -115544239990656 : ffff96e9c29e9880) 00000100:00000001:1.0:1784949289.397286:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306228779 00000100:00000010:1.0:1784949289.397295:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007f16776a. 00000100:00000040:2.0:1784949289.397296:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949289.397304:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949289.397315:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306228779 00000100:00000001:2.0:1784949289.397317:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:1.0:1784949289.397327:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000348d298. 00000100:00000040:2.0:1784949289.397328:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949289.397338:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000100:00000001:1.0:1784949289.397339:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000200:2.0:1784949289.397348:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289687424, offset 0 00000100:00000001:1.0:1784949289.397351:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306228779 00000400:00000200:2.0:1784949289.397360:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949289.397367:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1784949289.397369:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00100000:0.0:1784949289.397377:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:1.0:1784949289.397379:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000400:00000200:2.0:1784949289.397384:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949289.397388:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949289.397391:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949289.397396:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784949289.397397:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949289.397398:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.397404:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949289.397407:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949289.397408:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.397411:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949289.397418:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.397419:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949289.397419:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949289.397428:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949289.397429:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949289.397431:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949289.397438:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:0.0:1784949289.397439:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949289.397448:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3856:3856:241:4294967295] 192.168.202.137@tcp LPNI seq info [2464:2464:8:4294967295] 00000100:00000001:1.0:1784949289.397449:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949289.397454:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949289.397460:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.397471:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.397477:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949289.397484:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949289.397491:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784949289.397492:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:0.0:1784949289.397495:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949289.397502:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949289.397509:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949289.397510:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949289.397511:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c69adecd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289687552:192.168.202.137@tcp:400: 00000100:00100000:1.0:1784949289.397519:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289687296 00000100:00000001:0.0:1784949289.397525:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000800:00000010:2.0:1784949289.397525:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 02000000:00000001:1.0:1784949289.397526:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949289.397534:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949289.397536:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949289.397539:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949289.397541:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949289.397547:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:0.0:1784949289.397572:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1784949289.397574:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949289.397581:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 00000100:00100000:1.0:1784949289.397582:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289687296 00000020:00000001:1.0:1784949289.397609:0:342757:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1784949289.397610:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949289.397610:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000040:1.0:1784949289.397614:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949289.397621:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784949289.397622:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c8dbdfb2. 00000800:00000200:2.0:1784949289.397623:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 616 niov 1 nkiov 1 00000020:00000040:1.0:1784949289.397625:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949289.397642:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000010:0.0:1784949289.397645:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000020:00000001:1.0:1784949289.397646:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949289.397655:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949289.397656:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289687552, portal 10 00000100:00000001:1.0:1784949289.397657:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949289.397666:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1784949289.397666:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000020:00000010:1.0:1784949289.397668:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:0.0:1784949289.397672:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469569024 : -115544239982592 : ffff96e9c29eb800) 00000100:00000001:2.0:1784949289.397678:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306228779 00000100:00000001:1.0:1784949289.397679:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949289.397688:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949289.397691:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:2.0:1784949289.397692:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306228779 00000100:00000001:2.0:1784949289.397703:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306228779 00000020:00000010:1.0:1784949289.397707:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000bbdbdf4. 00000100:00000001:2.0:1784949289.397712:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306228779 00000100:00000001:0.0:1784949289.397714:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:1.0:1784949289.397718:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000319b9e18. 00000100:00000001:2.0:1784949289.397719:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306228779 00000100:00000040:0.0:1784949289.397728:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000010:1.0:1784949289.397731:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:2.0:1784949289.397731:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306228779 00000400:00000010:0.0:1784949289.397735:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f43fd4d9. 00000100:00000200:0.0:1784949289.397745:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289687552, offset 0 00000100:00000001:2.0:1784949289.397745:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306228779 00000100:00000040:1.0:1784949289.397746:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949289.397757:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949289.397758:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306228779 00000100:00000001:1.0:1784949289.397763:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949289.397768:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949289.397769:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949289.397770:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949289.397781:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.397786:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949289.397787:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784949289.397791:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949289.397802:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949289.397804:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949289.397806:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949289.397815:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1784949289.397816:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:2.0:1784949289.397824:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1784949289.397827:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784949289.397837:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784949289.397844:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:1.0:1784949289.397845:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949289.397846:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949289.397853:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949289.397854:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3857:3857:240:4294967295] 192.168.202.137@tcp LPNI seq info [2465:2465:7:4294967295] 00000100:00000001:2.0:1784949289.397860:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949289.397861:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 820 00000100:00000001:2.0:1784949289.397871:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949289.397872:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949289.397882:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529457129552 : -115544252422064 : ffff96e9c1e0e850) 00000400:00000200:0.0:1784949289.397891:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:1.0:1784949289.397894:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009b82ab0d x1871651289687296/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:195/0 lens 224/0 e 0 to 0 dl 1784949300 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949289.397897:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949289.397905:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949289.397911:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000200:3.0:1784949289.397958:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949289.397971:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000100:00000001:1.0:1784949289.397972:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949289.397978:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:1.0:1784949289.397984:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949289.397986:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1784949289.397988:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00100000:1.0:1784949289.398006:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009b82ab0d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289687296:12345-0@lo:400: 00000100:00000200:1.0:1784949289.398015:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289687296 00000400:00000200:3.0:1784949289.398015:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784949289.398023:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:3.0:1784949289.398029:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000020:00000001:1.0:1784949289.398039:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:3.0:1784949289.398040:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000020:00000001:1.0:1784949289.398045:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949289.398051:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784949289.398052:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949289.398058:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949289.398062:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000200:3.0:1784949289.398064:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949289.398066:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949289.398075:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949289.398081:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949289.398086:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949289.398091:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784949289.398092:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:0.0:1784949289.398093:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949289.398099:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949314 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949289.398100:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949289.398106:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:0.0:1784949289.398108:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949289.398115:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000061405111. 00000800:00000001:0.0:1784949289.398117:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:1.0:1784949289.398124:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949289.398126:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949289.398127:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.398130:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949289.398137:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1784949289.398139:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949289.398149:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00010000:00000040:1.0:1784949289.398149:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289687296 00010000:00000001:1.0:1784949289.398158:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949289.398166:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009b82ab0d x1871651289687296/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:195/0 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949289.398173:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643e580 00000400:00000100:0.0:1784949289.398190:0:337114:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289687424 offset 0 length 520: 4 00010000:00000001:1.0:1784949289.398191:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949289.398197:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949289.398199:0:337114:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000182da6a8 is in inconsistent state, don't perform health checking (-2, 0) 00000100:00001000:1.0:1784949289.398205:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=11 v=5 (1 1 1 1) 00000400:00000200:0.0:1784949289.398206:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:1.0:1784949289.398218:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:0.0:1784949289.398222:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949289.398223:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949289.398229:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:0.0:1784949289.398232:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949289.398233:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949289.398238:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949289.398241:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949289.398243:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000001:0.0:1784949289.398248:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949289.398249:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949289.398257:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000100:00000200:1.0:1784949289.398263:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289687296, offset 224 00000400:00000200:1.0:1784949289.398268:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000010:0.0:1784949289.398272:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000635133da. 00000400:00000200:1.0:1784949289.398281:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:0.0:1784949289.398289:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949289.398294:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e500 00000400:00000200:1.0:1784949289.398305:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9549 [1] + 224 00000800:00000200:0.0:1784949289.398314:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000200:1.0:1784949289.398317:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1784949289.398326:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000635133da type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1784949289.398328:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949289.398340:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949289.398344:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.398354:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949289.398365:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949289.398371:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2245 usecs (3 jiffies) 00000100:00000001:0.0:1784949289.398381:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306228741 00000100:00000001:0.0:1784949289.398388:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306228779 00000100:00000001:0.0:1784949289.398391:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306228741 00000100:00000001:0.0:1784949289.398395:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306228779 00000100:00000001:1.0:1784949289.398395:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949289.398399:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306228741 00000100:00000001:0.0:1784949289.398404:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306228779 00000400:00000200:1.0:1784949289.398407:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000100:00000001:0.0:1784949289.398408:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306228741 00000400:00000010:1.0:1784949289.398414:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000100:00000001:0.0:1784949289.398416:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306228779 00000100:00000001:0.0:1784949289.398424:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949289.398425:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949289.398430:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949289.398431:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.398439:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.398441:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.398444:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949289.398450:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784949289.398456:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949289.398459:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949289.398465:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949289.398471:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009b82ab0d x1871651289687296/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:195/0 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949289.398484:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009b82ab0d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289687296:12345-0@lo:400: Request processed in 511us (1467us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949289.398499:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 820 00000800:00000200:3.0:1784949289.398504:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:1.0:1784949289.398505:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949289.398513:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000010:3.0:1784949289.398515:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000635133da. 00000100:00000001:1.0:1784949289.398519:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:3.0:1784949289.398525:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1784949289.398527:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000319b9e18. 00000100:00000001:0.0:1784949289.398528:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949289.398538:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949289.398539:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:1.0:1784949289.398540:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000400:00000001:0.0:1784949289.398573:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:1.0:1784949289.398574:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000bbdbdf4. 00000400:00000001:0.0:1784949289.398583:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:1.0:1784949289.398585:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000010:0.0:1784949289.398591:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000aba79327. 02000000:00000001:1.0:1784949289.398594:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949289.398606:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000061405111. 00000400:00000200:0.0:1784949289.398616:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 02000000:00000001:1.0:1784949289.398616:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949289.398622:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949289.398624:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000200:1.0:1784949289.398629:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784949289.398635:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000010:1.0:1784949289.398637:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a626aa3a. 00000400:00000200:3.0:1784949289.398637:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f43fd4d9 00000400:00000010:3.0:1784949289.398645:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f43fd4d9. 00000100:00000200:0.0:1784949289.398652:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949289.398656:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:1.0:1784949289.398661:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000302f9c9c. 00000800:00000001:2.0:1784949289.398665:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784949289.398669:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949289.398681:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949289.398707:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:0.0:1784949289.398710:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949289.398718:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949289.398722:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949289.398734:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949289.398765:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949289.398766:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.398767:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949289.398776:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949289.398778:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.398786:0:337120:0:(client.c:1580:after_reply()) Process entered 00000400:00000200:2.0:1784949289.398794:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e600 02000000:00000001:0.0:1784949289.398797:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949289.398804:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949289.398810:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949289.398815:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 179344 02000000:00000001:0.0:1784949289.398828:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949289.398836:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=11 v=5 (5 5 5 5) 00000800:00000001:2.0:1784949289.398838:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:0.0:1784949289.398843:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=11 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949289.398849:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949289.398851:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949289.398855:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784949289.398858:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.398861:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949289.398864:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949289.398871:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784949289.398872:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949289.398888:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949289.398895:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949289.398901:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784949289.398902:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000200:2.0:1784949289.398945:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:2.0:1784949289.398952:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009e01665e. 00000100:00000001:0.0:1784949289.398957:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949289.398960:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000009e01665e x1871651289687552 msgsize 224 00000100:00000040:0.0:1784949289.398967:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949289.398982:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00100000:0.0:1784949289.398996:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289687296:0@lo:400: 00000100:00000001:0.0:1784949289.399003:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.399005:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949289.399008:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784949289.399009:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ca646268 x1871651289687296/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949289.399019:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949289.399021:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:2.0:1784949289.399028:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949289.399038:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949289.399043:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000229221cc. 02000000:00000001:0.0:1784949289.399050:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949289.399052:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1784949289.399054:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000040:0.0:1784949289.399055:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00100000:1.0:1784949289.399062:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289687552 00000020:00000001:0.0:1784949289.399063:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949289.399068:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000057c7ab4. 02000000:00000001:1.0:1784949289.399070:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:0.0:1784949289.399074:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1784949289.399075:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949289.399080:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1784949289.399081:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949289.399086:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ca646268. 02000000:00000001:1.0:1784949289.399088:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.399095:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00100000:1.0:1784949289.399096:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289687552 00000100:00000001:0.0:1784949289.399100:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949289.399102:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1784949289.399106:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784949289.399107:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949289.399115:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949289.399118:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:1.0:1784949289.399119:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:0.0:1784949289.399124:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949289.399126:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949289.399128:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1784949289.399134:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949289.399135:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.399140:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949289.399146:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:1.0:1784949289.399150:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949289.399153:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000020:00000010:1.0:1784949289.399158:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000bbdbdf4. 00000020:00000010:1.0:1784949289.399163:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000319b9e18. 00000020:00000010:1.0:1784949289.399166:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949289.399173:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949289.399176:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949289.399177:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949289.399180:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949289.399185:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949289.399188:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949289.399190:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.399206:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949289.399215:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949289.399217:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949289.399222:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 796 00000100:00000040:1.0:1784949289.399226:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949289.399228:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469578240 : -115544239973376 : ffff96e9c29edc00) 00000100:00000040:1.0:1784949289.399234:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009e01665e x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:195/0 lens 224/0 e 0 to 0 dl 1784949300 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949289.399244:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949289.399247:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:1.0:1784949289.399250:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949289.399252:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00100000:1.0:1784949289.399256:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009e01665e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289687552:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949289.399260:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289687552 00000020:00000001:1.0:1784949289.399263:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949289.399266:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949289.399267:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949289.399269:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949289.399271:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949289.399274:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949289.399277:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949289.399279:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949289.399281:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949289.399283:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949289.399286:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949289.399290:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949289.399292:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949289.399295:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000061405111. 02000000:00000001:1.0:1784949289.399297:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.399299:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949289.399301:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949289.399303:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289687552 00010000:00000001:1.0:1784949289.399306:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949289.399309:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009e01665e x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:195/0 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949289.399317:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949289.399319:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949289.399333:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=6 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949289.399336:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949289.399339:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949289.399342:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949289.399344:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949289.399346:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949289.399348:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949289.399350:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949289.399353:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000100:00000200:1.0:1784949289.399356:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289687552, offset 224 00000400:00000200:1.0:1784949289.399361:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949289.399370:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949289.399375:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3858:3858:241:4294967295] 192.168.202.137@tcp LPNI seq info [2466:2466:8:4294967295] 00000400:00000200:1.0:1784949289.399386:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949289.399392:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949289.399396:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ca1124f5. 00000800:00000200:1.0:1784949289.399402:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949289.399408:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949289.399411:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ca1124f5 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949289.399426:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949289.399429:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949289.399432:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949289.399433:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949289.399435:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949289.399439:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009e01665e x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:195/0 lens 224/224 e 0 to 0 dl 1784949300 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949289.399448:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009e01665e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289687552:12345-192.168.202.137@tcp:400: Request processed in 203us (488us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949289.399456:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 796 00000100:00000040:1.0:1784949289.399459:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949289.399461:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949289.399463:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949289.399465:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000319b9e18. 00000020:00000010:1.0:1784949289.399468:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949289.399471:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000bbdbdf4. 00000020:00000040:1.0:1784949289.399474:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949289.399477:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949289.399602:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949289.399609:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ca1124f5. 00000400:00000200:0.0:1784949289.399615:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949289.399623:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949289.399630:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000400:00000010:0.0:1784949289.399632:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000100:00000001:0.0:1784949289.399638:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949289.399640:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949289.399646:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949289.399658:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949289.399662:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949289.399664:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949289.399672:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949289.399684:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e600 00000400:00000200:3.0:1784949289.399690:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9559 [1] + 224 00000800:00000001:3.0:1784949289.399697:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949289.399701:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949289.399704:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949289.399708:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949289.399713:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949289.399717:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949289.399728:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949289.399749:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949289.399756:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949289.399759:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.399783:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949289.399788:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949289.399793:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949289.399795:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949289.399798:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c8dbdfb2. 00000400:00000200:0.0:1784949289.399802:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000400:00000010:0.0:1784949289.399804:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000100:00000001:0.0:1784949289.399807:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949289.399813:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949289.399834:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.399843:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949289.399845:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.399848:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949289.399850:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949289.399852:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949289.399854:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949289.399856:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949289.399862:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=6 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949289.399866:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=6 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949289.399869:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949289.399871:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949289.399874:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949289.399877:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949289.399886:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.399890:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949289.399892:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949289.399894:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949289.399895:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949289.399898:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.399947:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949289.399951:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949289.399953:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949289.399957:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949289.399967:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c69adecd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289687552:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949289.399973:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.399975:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949289.399977:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c69adecd x1871651289687552/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949289.399985:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949289.399987:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949289.399989:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949289.399994:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949289.399996:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949289.399997:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949289.400000:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949289.400003:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003a8a98bc. 02000000:00000001:0.0:1784949289.400007:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949289.400009:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949289.400011:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c69adecd. 00000100:00000001:0.0:1784949289.400017:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949289.400019:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.400021:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949289.400026:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949289.400028:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949289.400030:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949289.400032:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949289.440048:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306228779, at 4306228780 00000100:00000040:1.0:1784949289.440056:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306228779, at 4306228780 00000100:00000040:3.0:1784949289.440056:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306228779, at 4306228780 00000100:00000010:2.0:1784949289.440064:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000348d298. 00000100:00000040:0.0:1784949289.440073:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306228779, at 4306228780 00000100:00000010:1.0:1784949289.440075:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007f16776a. 00000100:00000010:3.0:1784949289.440078:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:0.0:1784949289.440092:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001a2e8205. 00010000:00000001:2.0:1784949290.228038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949290.228048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949290.228056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949290.228057:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949290.228158:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949290.228164:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949290.228169:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949291.252039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949291.252049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949291.252057:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949291.252061:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949291.252065:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949291.252069:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949291.252073:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949291.252076:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949291.252078:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949291.252081:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949291.252203:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949291.252210:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949291.252214:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000010:3.0:1784949291.635997:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000043ce20a5. 00000400:00000010:3.0:1784949291.636015:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000039173808. 00000400:00000010:3.0:1784949291.636018:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006e91e08a. 00000400:00000010:3.0:1784949291.636020:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007d16b15b. 00000400:00000010:3.0:1784949291.636022:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000033f377e8. 00000400:00000010:3.0:1784949291.636024:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000081a06abd. 00000400:00000010:3.0:1784949291.636027:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000007b996847. 00000400:00000010:3.0:1784949291.636029:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001721bb7a. 00000400:00000010:3.0:1784949291.636031:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000339a3bc6. 00000400:00000010:3.0:1784949291.636033:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000059f24104. 00000400:00000010:3.0:1784949291.636053:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000044d70af9. 00000400:00000010:3.0:1784949291.636056:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004447c62a. 00000400:00000010:3.0:1784949291.636060:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009ea2352f. 00000400:00000010:3.0:1784949291.636063:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009b59e4ef. 00000400:00000010:3.0:1784949291.636067:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000051d06b10. 00000400:00000010:3.0:1784949291.636070:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000812f4e7a. 00000400:00000010:3.0:1784949291.636073:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000040c3d822. 00000400:00000010:3.0:1784949291.636077:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000005af93f5d. 00000400:00000010:3.0:1784949291.636080:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000092c6c4bf. 00000400:00000010:3.0:1784949291.636083:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a2592785. 00000400:00000010:3.0:1784949291.636086:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000002d887daf. 00000400:00000010:3.0:1784949291.636090:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000db9e65ed. 00000400:00000010:3.0:1784949291.636093:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000096223606. 00000400:00000010:3.0:1784949291.636097:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f50feabf. 00000400:00000010:3.0:1784949291.636101:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000627f8b94. 00000400:00000010:3.0:1784949291.636120:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000097ad9460. 00000400:00000010:3.0:1784949291.636125:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000c0f241eb. 00000400:00000010:3.0:1784949291.636129:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000091a5155f. 00000400:00000010:3.0:1784949291.636132:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000225c88a3. 00000400:00000010:3.0:1784949291.636137:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e69d59cc. 00000400:00000010:3.0:1784949291.636143:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000fa5520c4. 00000400:00000010:3.0:1784949291.636146:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b5ecaa25. 00000400:00000010:3.0:1784949291.636150:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e07f0bcd. 00000400:00000010:3.0:1784949291.636154:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000061508776. 00010000:00000001:2.0:1784949292.276013:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949292.276022:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949292.276030:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949292.276031:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949292.276109:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949292.276116:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949292.276121:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1784949292.660045:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:2.0:1784949292.660054:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:2.0:1784949292.660065:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949292.660072:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949292.660080:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11565 health :900 00000400:00000200:2.0:1784949292.660086:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949292.660092:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:2.0:1784949292.660097:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:3.0:1784949292.660163:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949292.660174:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:3.0:1784949292.660182:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949292.660186:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949292.660189:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:3.0:1784949292.660194:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000400:00000200:3.0:1784949292.660202:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949292.660208:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:3.0:1784949292.660214:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 00000400:00000200:3.0:1784949292.660225:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:3.0:1784949292.660230:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:3.0:1784949292.660236:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00010000:00000001:2.0:1784949293.299972:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949293.299979:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949293.299984:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949293.299985:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949293.300056:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949293.300061:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949293.300063:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:1.0:1784949293.684120:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949293.684133:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000400:00000010:1.0:1784949293.684148:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000bbdbdf4. 00000400:00000010:1.0:1784949293.684156:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000400:00000010:1.0:1784949293.684165:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000038468d93. 00000400:00000200:1.0:1784949293.684171:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949293.684190:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949293.684200:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949293.684205:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3859:3859:242:4294967295] 192.168.202.137@tcp LPNI seq info [2467:2467:8:4294967295] 00000400:00000200:1.0:1784949293.684213:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949293.684223:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949293.684230:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949293.684235:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000094ed938f. 00000800:00000200:1.0:1784949293.684243:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949293.684255:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949293.684260:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000094ed938f type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949293.684288:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11567 health :900 00000800:00000001:0.0:1784949293.684485:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949293.684489:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949293.684499:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000094ed938f. 00000800:00000001:0.0:1784949293.684503:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949293.684507:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949293.684510:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949293.684550:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:0.0:1784949293.684551:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949293.684562:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949293.684571:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949293.684577:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:0.0:1784949293.684598:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949293.684612:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:0.0:1784949293.684626:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949293.684633:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3860:3860:242:4294967295] 192.168.202.137@tcp LPNI seq info [2468:2468:8:4294967295] 00000400:00000200:0.0:1784949293.684643:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:0.0:1784949293.684649:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949293.684654:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003a8a98bc. 00000800:00000200:0.0:1784949293.684660:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949293.684666:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949293.684669:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003a8a98bc type 3, nob 144 niov 1 nkiov 1 00000800:00000200:0.0:1784949293.684754:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949293.684759:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003a8a98bc. 00000800:00000001:3.0:1784949293.684763:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949293.684766:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949293.684776:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949293.684781:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949293.684791:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949293.684803:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949293.684808:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949293.684816:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949293.684834:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949293.684853:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x956d 00000800:00000001:3.0:1784949293.684861:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949293.684866:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949293.684869:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949293.684874:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949293.684880:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:3.0:1784949293.684884:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000400:00000200:3.0:1784949293.684889:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949293.684893:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000010:3.0:1784949293.684897:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000bbdbdf4. 00000800:00000001:3.0:1784949293.684937:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949293.684941:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949294.324041:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949294.324052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949294.324060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949294.324062:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949294.324151:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949294.324157:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949294.324162:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949294.516050:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949294.516067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949294.516071:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949294.516078:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949294.516082:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949294.516086:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.516090:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949294.516115:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949294.516126:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949294.516139:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c69adecd. 00000020:00000040:0.0:1784949294.516148:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949294.516155:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949294.516159:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949294.516166:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949294.516177:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003a8a98bc. 00000100:00000001:0.0:1784949294.516187:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949294.516199:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949294.516215:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949294.516219:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949294.516227:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c69adecd] to pc [ptlrpcd_00_03+3] req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949294.516267:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949294.516271:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949294.516277:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949294.516281:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949294.516286:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949294.516291:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ca646268. 00000020:00000040:0.0:1784949294.516295:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949294.516299:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949294.516303:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949294.516307:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949294.516313:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000057c7ab4. 00000100:00000001:2.0:1784949294.516316:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949294.516319:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.516323:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949294.516328:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949294.516331:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949294.516341:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949294.516342:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949294.516354:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949294.516355:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949294.516360:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949294.516376:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca646268] to pc [ptlrpcd_00_00+0] req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949294.516377:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949294.516384:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949294.516395:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949294.516402:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949294.516413:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949294.516420:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c69adecd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289687680:0@lo:400: 00000100:00000001:2.0:1784949294.516427:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949294.516436:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949294.516439:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949294.516441:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949294.516444:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784949294.516453:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11571) 02000000:00000010:2.0:1784949294.516455:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000059e5b358. 00000100:00000001:3.0:1784949294.516460:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949294.516464:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.516467:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949294.516473:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949294.516476:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1784949294.516477:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e2d054c8. 00000100:00000001:0.0:1784949294.516483:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949294.516486:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949294.516490:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000100:00000200:2.0:1784949294.516507:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289687680, portal 25 00000100:00000001:3.0:1784949294.516508:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949294.516539:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949294.516542:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.516544:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469569024 : -115544239982592 : ffff96e9c29eb800) 02000000:00000001:3.0:1784949294.516555:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1784949294.516561:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949294.516564:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949294.516574:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949294.516587:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289687808:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949294.516593:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949294.516596:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1784949294.516603:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784949294.516604:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949294.516609:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949294.516610:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 02000000:00000001:3.0:1784949294.516612:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000200:2.0:1784949294.516619:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289687680, offset 0 02000000:00000010:3.0:1784949294.516623:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 02000000:00000001:3.0:1784949294.516629:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949294.516637:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784949294.516643:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000400:00000010:3.0:1784949294.516656:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000400:00000200:2.0:1784949294.516664:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:3.0:1784949294.516668:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289687808, portal 10 00000100:00000001:3.0:1784949294.516674:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949294.516677:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469564544 : -115544239987072 : ffff96e9c29ea680) 00000400:00000200:2.0:1784949294.516688:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e680 00000100:00000040:3.0:1784949294.516691:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949294.516710:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8b5d [2] + 0 00000100:00000001:3.0:1784949294.516713:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:2.0:1784949294.516722:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f4e53d4f. 00000100:00000040:3.0:1784949294.516724:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949294.516735:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c1d3a6d6 00000400:00000010:3.0:1784949294.516736:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000c2c66d. 00000100:00000200:3.0:1784949294.516746:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289687808, offset 0 00000400:00000200:2.0:1784949294.516752:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949294.516760:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949294.516767:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949294.516781:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949294.516788:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949294.516795:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949294.516800:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949294.516805:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949294.516811:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3861:3861:242:4294967295] 192.168.202.137@tcp LPNI seq info [2469:2469:8:4294967295] 00000400:00000200:3.0:1784949294.516826:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949294.516834:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949294.516836:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c1d3a6d6 00000400:00000010:2.0:1784949294.516843:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000c1d3a6d6. 00000800:00000010:3.0:1784949294.516844:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000100:00000001:2.0:1784949294.516855:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:3.0:1784949294.516862:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:2.0:1784949294.516863:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949294.516875:0:337122:0:(events.c:337:request_in_callback()) incoming req@0000000068fd7440 x1871651289687680 msgsize 224 00000800:00000200:3.0:1784949294.516882:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:2.0:1784949294.516894:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000200:3.0:1784949294.516895:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:2.0:1784949294.516900:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949294.516955:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949294.516960:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949294.516966:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000010:2.0:1784949294.516972:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000100:00000040:3.0:1784949294.516973:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2246 usecs (3 jiffies) 00000100:00000001:2.0:1784949294.516978:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949294.516986:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306233860 00000100:00000200:2.0:1784949294.516988:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784949294.517002:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:2.0:1784949294.517007:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949294.517015:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949294.517020:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949294.517034:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.517039:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949294.517043:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306233860 00000100:00000001:2.0:1784949294.517047:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949294.517053:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949294.517056:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306233899 00000100:00000010:3.0:1784949294.517058:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00100000:0.0:1784949294.517060:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289687680 02000000:00000001:0.0:1784949294.517067:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949294.517072:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949294.517074:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306233860 00000100:00000001:3.0:1784949294.517074:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949294.517076:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949294.517091:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.517096:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306233899 00000100:00000001:3.0:1784949294.517097:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306233860 00000100:00100000:0.0:1784949294.517098:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289687680 00000020:00000001:0.0:1784949294.517108:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000010:3.0:1784949294.517110:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000001:2.0:1784949294.517111:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306233860 00000020:00000040:0.0:1784949294.517113:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949294.517123:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949294.517127:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306233899 00000100:00000001:3.0:1784949294.517135:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306233860 00000100:00000001:2.0:1784949294.517141:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306233860 00000100:00000010:3.0:1784949294.517143:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:3.0:1784949294.517153:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000800:00000200:1.0:1784949294.517156:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949294.517157:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306233899 00000100:00000001:3.0:1784949294.517161:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949294.517170:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949294.517171:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000100:00000001:3.0:1784949294.517175:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.517184:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949294.517190:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949294.517195:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949294.517202:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:1.0:1784949294.517211:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949294.517216:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784949294.517221:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000c2c66d 00000100:00000001:3.0:1784949294.517222:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:1.0:1784949294.517230:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000c2c66d. 00000100:00000001:3.0:1784949294.517233:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949294.517241:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784949294.517243:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949294.517245:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949294.517251:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000200:1.0:1784949294.517256:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949294.517260:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784949294.517271:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949294.517282:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949294.517285:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1784949294.517290:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000040:1.0:1784949294.517293:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949294.517299:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949294.517312:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000800:00000001:2.0:1784949294.517313:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949294.517324:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949294.517328:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000100:00000001:1.0:1784949294.517329:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949294.517334:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306233860 00000800:00000001:2.0:1784949294.517335:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949294.517339:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000734334de. 00000800:00000001:2.0:1784949294.517346:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:0.0:1784949294.517353:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002ffa19fa. 00000100:00000001:3.0:1784949294.517354:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000001:2.0:1784949294.517357:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949294.517367:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949294.517374:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949294.517376:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949294.517383:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:2.0:1784949294.517385:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949294.517388:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949294.517390:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949294.517397:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.517404:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1784949294.517411:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e700 00000100:00000001:0.0:1784949294.517412:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.517413:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784949294.517429:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949294.517434:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949294.517438:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 179568 00000100:00000001:3.0:1784949294.517439:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949294.517448:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949294.517452:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949294.517454:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949294.517458:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949294.517460:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:2.0:1784949294.517468:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949294.517476:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:0.0:1784949294.517477:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 821 00000100:00000040:0.0:1784949294.517488:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:2.0:1784949294.517496:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949294.517497:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770431568 : -115539939120048 : ffff96eac2f8ac50) 00000100:00000001:2.0:1784949294.517537:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:0.0:1784949294.517538:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000068fd7440 x1871651289687680/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:200/0 lens 224/0 e 0 to 0 dl 1784949305 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949294.517545:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000010:2.0:1784949294.517557:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b4a9b791. 00000100:00000001:0.0:1784949294.517562:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784949294.517564:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000b4a9b791 x1871651289687808 msgsize 224 00000020:00000010:0.0:1784949294.517569:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:0.0:1784949294.517579:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784949294.517581:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000010:0.0:1784949294.517586:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00100000:0.0:1784949294.517598:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000068fd7440 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289687680:12345-0@lo:400: 00000100:00000001:2.0:1784949294.517602:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:0.0:1784949294.517605:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289687680 00000020:00000001:0.0:1784949294.517612:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000001:2.0:1784949294.517615:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949294.517617:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949294.517623:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949294.517625:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949294.517627:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949294.517633:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949294.517638:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:0.0:1784949294.517640:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949294.517646:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949294.517650:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289687808 00000020:00000001:0.0:1784949294.517653:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000001:3.0:1784949294.517660:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:0.0:1784949294.517661:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:3.0:1784949294.517669:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:0.0:1784949294.517670:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.517678:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949294.517679:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949294.517685:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949294.517690:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949294.517692:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:3.0:1784949294.517701:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289687808 02000000:00000010:0.0:1784949294.517704:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003e4182b6. 00000020:00000001:3.0:1784949294.517708:0:342762:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1784949294.517711:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949294.517714:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:0.0:1784949294.517717:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949294.517722:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:0.0:1784949294.517724:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949294.517730:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000040:0.0:1784949294.517731:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289687680 00010000:00000001:0.0:1784949294.517744:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:3.0:1784949294.517744:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000200:0.0:1784949294.517754:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000068fd7440 x1871651289687680/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:200/0 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949294.517756:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949294.517771:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:0.0:1784949294.517775:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949294.517780:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949294.517780:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006e48452f. 00000100:00000001:3.0:1784949294.517791:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00001000:0.0:1784949294.517792:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=16 v=5 (1 1 1 1) 00000020:00000010:3.0:1784949294.517799:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006e48452f. 00000100:00000001:0.0:1784949294.517803:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949294.517809:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000010:3.0:1784949294.517810:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000035b54a67. 00000100:00000001:0.0:1784949294.517814:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000010:3.0:1784949294.517819:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007c705757. 02000000:00000001:0.0:1784949294.517821:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949294.517826:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949294.517831:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949294.517833:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949294.517840:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:3.0:1784949294.517847:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:0.0:1784949294.517848:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000100:00000001:3.0:1784949294.517858:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000200:0.0:1784949294.517861:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289687680, offset 224 00000100:00000001:3.0:1784949294.517863:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949294.517869:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949294.517874:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949294.517881:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949294.517887:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000400:00000200:0.0:1784949294.517894:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949294.517894:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949294.517944:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e680 00000400:00000200:0.0:1784949294.517954:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9571 [1] + 224 00000100:00000001:3.0:1784949294.517959:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949294.517964:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949294.517968:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949294.517975:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:0.0:1784949294.517977:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949294.517988:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949294.517989:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784949294.517991:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 797 00000100:00000001:2.0:1784949294.517998:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:0.0:1784949294.517999:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949294.518001:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949294.518005:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.518012:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469573760 : -115544239977856 : ffff96e9c29eca80) 00000100:00000001:2.0:1784949294.518016:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949294.518031:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000b4a9b791 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:200/0 lens 224/0 e 0 to 0 dl 1784949305 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949294.518047:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949294.518056:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949294.518064:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006e48452f. 00000100:00000001:3.0:1784949294.518070:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949294.518073:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006e48452f. 00000100:00100000:3.0:1784949294.518079:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000b4a9b791 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289687808:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949294.518084:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:3.0:1784949294.518085:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289687808 00000020:00000001:3.0:1784949294.518091:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784949294.518096:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000020:00000001:3.0:1784949294.518098:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949294.518104:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949294.518106:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000020:00000001:3.0:1784949294.518111:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949294.518116:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:3.0:1784949294.518116:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784949294.518122:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949294.518123:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:3.0:1784949294.518124:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:2.0:1784949294.518132:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949294.518133:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949294.518135:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784949294.518144:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1784949294.518145:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:0.0:1784949294.518147:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:2.0:1784949294.518154:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:0.0:1784949294.518155:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949294.518155:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949294.518164:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949294.518165:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e2d054c8. 00000020:00000001:3.0:1784949294.518166:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949294.518175:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:3.0:1784949294.518178:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:2.0:1784949294.518180:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000100:00000040:0.0:1784949294.518185:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000068fd7440 x1871651289687680/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:200/0 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949294.518187:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000010:2.0:1784949294.518188:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 02000000:00000001:3.0:1784949294.518197:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949294.518200:0:337122:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:3.0:1784949294.518209:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007afc0d3b. 00000100:00000200:2.0:1784949294.518212:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949294.518219:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000068fd7440 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289687680:12345-0@lo:400: Request processed in 650us (1336us total) trans 0 rc 0/0 02000000:00000001:3.0:1784949294.518222:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.518235:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949294.518250:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949294.518254:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 821 00010000:00000040:3.0:1784949294.518262:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289687808 00000100:00000040:0.0:1784949294.518266:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:2.0:1784949294.518272:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949294.518273:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949294.518274:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949294.518283:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00010000:00000200:3.0:1784949294.518290:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000b4a9b791 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:200/0 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949294.518292:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000734334de. 00000100:00000001:2.0:1784949294.518298:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784949294.518303:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002ffa19fa. 00000100:00000001:2.0:1784949294.518305:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949294.518315:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000100:00000001:2.0:1784949294.518316:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949294.518324:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:0.0:1784949294.518327:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784949294.518330:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00010000:00000001:3.0:1784949294.518335:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949294.518336:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949294.518339:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:3.0:1784949294.518342:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949294.518345:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949294.518346:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000003e4182b6. 00000100:00001000:3.0:1784949294.518351:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=11 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949294.518358:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:2.0:1784949294.518360:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=16 v=5 (5 5 5 5) 00000100:00000001:3.0:1784949294.518361:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949294.518364:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949294.518373:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00001000:2.0:1784949294.518376:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=16 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949294.518377:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949294.518379:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000010:0.0:1784949294.518390:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c8dbdfb2. 02000000:00000001:3.0:1784949294.518392:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784949294.518393:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000001:3.0:1784949294.518400:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.518403:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949294.518405:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d6bfc0b3. 00000100:00000001:3.0:1784949294.518407:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1784949294.518409:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949294.518416:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949294.518422:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949294.518425:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949294.518431:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000029a9ebd3. 00000100:00000040:2.0:1784949294.518433:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949294.518443:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289687808, offset 224 00000100:00000001:2.0:1784949294.518457:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949294.518464:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949294.518466:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949294.518472:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949294.518479:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949294.518486:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949294.518503:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3862:3862:242:4294967295] 192.168.202.137@tcp LPNI seq info [2470:2470:8:4294967295] 00000100:00100000:2.0:1784949294.518541:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c69adecd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289687680:0@lo:400: 00000400:00000200:3.0:1784949294.518559:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784949294.518561:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949294.518571:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784949294.518577:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784949294.518580:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c69adecd x1871651289687680/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1784949294.518587:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000100:00000001:2.0:1784949294.518600:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000200:3.0:1784949294.518603:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:2.0:1784949294.518606:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949294.518611:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000059e5b358. 00000800:00000200:3.0:1784949294.518618:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000001:2.0:1784949294.518621:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000200:3.0:1784949294.518625:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000020:00000001:2.0:1784949294.518628:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949294.518633:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949294.518637:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949294.518639:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003a8a98bc. 02000000:00000001:2.0:1784949294.518644:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949294.518646:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949294.518648:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c69adecd. 00000100:00000001:3.0:1784949294.518650:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.518653:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:3.0:1784949294.518659:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949294.518660:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949294.518669:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:2.0:1784949294.518670:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949294.518677:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949294.518682:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1784949294.518683:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784949294.518691:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949294.518697:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b4a9b791 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:200/0 lens 224/224 e 0 to 0 dl 1784949305 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949294.518698:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949294.518705:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949294.518721:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000b4a9b791 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289687808:12345-192.168.202.137@tcp:400: Request processed in 664us (1155us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949294.518734:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 797 00000100:00000040:3.0:1784949294.518738:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949294.518743:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949294.518746:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949294.518750:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007c705757. 00000020:00000010:3.0:1784949294.518756:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949294.518760:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000035b54a67. 00000020:00000040:3.0:1784949294.518768:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949294.518772:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949294.518831:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949294.518837:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:0.0:1784949294.518842:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949294.518848:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949294.518852:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000029a9ebd3 00000400:00000010:0.0:1784949294.518855:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000029a9ebd3. 00000100:00000001:0.0:1784949294.518859:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949294.518861:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949294.518868:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949294.518875:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949294.518895:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949294.518898:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949294.518905:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949294.518915:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e700 00000400:00000200:0.0:1784949294.518923:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9579 [1] + 224 00000800:00000001:0.0:1784949294.518929:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949294.518978:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949294.518982:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949294.518987:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949294.518993:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949294.518998:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949294.519011:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949294.519029:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949294.519035:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949294.519038:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949294.519082:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949294.519086:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949294.519091:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949294.519093:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949294.519096:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000400:00000200:3.0:1784949294.519103:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000400:00000010:3.0:1784949294.519105:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000100:00000001:3.0:1784949294.519108:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949294.519115:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949294.519126:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949294.519136:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949294.519137:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949294.519140:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949294.519142:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949294.519144:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949294.519146:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949294.519148:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949294.519154:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=11 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949294.519157:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=11 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949294.519160:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949294.519162:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949294.519165:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949294.519169:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949294.519178:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.519182:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949294.519184:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949294.519187:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949294.519189:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949294.519193:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949294.519206:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949294.519210:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949294.519213:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949294.519219:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949294.519234:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289687808:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949294.519242:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949294.519246:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949294.519251:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ca646268 x1871651289687808/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949294.519264:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949294.519267:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949294.519271:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 02000000:00000001:3.0:1784949294.519276:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949294.519279:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949294.519281:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949294.519285:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949294.519288:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000057c7ab4. 02000000:00000001:3.0:1784949294.519293:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949294.519295:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949294.519298:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ca646268. 00000100:00000001:3.0:1784949294.519305:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949294.519307:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949294.519311:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949294.519317:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949294.519320:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949294.519324:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949294.519327:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949294.560034:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306233899, at 4306233900 00000100:00000040:3.0:1784949294.560050:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306233899, at 4306233900 00000100:00000010:1.0:1784949294.560054:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000040:2.0:1784949294.560057:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306233899, at 4306233900 00000100:00000010:3.0:1784949294.560066:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000010:2.0:1784949294.560072:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000040:0.0:1784949294.560161:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306233899, at 4306233900 00000100:00000010:0.0:1784949294.560171:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00010000:00000001:2.0:1784949295.348012:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949295.348022:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949295.348031:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949295.348033:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949295.348168:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949295.348174:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949295.348179:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949295.540039:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949295.540054:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949295.540057:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949295.540063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949295.540068:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949295.540088:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949295.540094:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949296.372032:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949296.372042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949296.372050:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949296.372052:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949296.372179:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949296.372185:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949296.372189:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949296.564030:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949296.564050:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949296.564056:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949296.564064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949296.564069:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949296.564075:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949296.564080:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949297.396017:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949297.396027:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949297.396035:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949297.396038:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949297.396138:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949297.396142:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949297.396146:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949297.588003:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949297.588021:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949297.588023:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949297.588029:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949297.588033:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949297.588037:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949297.588040:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949297.999391:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949297.999413:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949297.999418:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949297.999420:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949298.420101:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949298.420111:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949298.420119:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949298.420121:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949298.420240:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949298.420250:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949298.420257:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949298.612042:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949298.612056:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949298.612058:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949298.612064:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949298.612068:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949298.612072:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949298.612075:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949299.444031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949299.444040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949299.444048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949299.444050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949299.444180:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949299.444186:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949299.444190:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949299.636186:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1784949299.636203:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949299.636204:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949299.636209:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949299.636219:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949299.636224:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949299.636225:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949299.636233:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.636237:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949299.636239:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ca646268. 00000020:00000040:0.0:1784949299.636254:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949299.636265:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949299.636271:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949299.636281:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949299.636297:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d104522b. 00000100:00000001:0.0:1784949299.636311:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949299.636328:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949299.636352:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949299.636357:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949299.636369:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca646268] to pc [ptlrpcd_00_01+1] req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949299.636433:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949299.636440:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949299.636448:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949299.636455:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949299.636466:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949299.636556:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949299.636561:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949299.636563:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949299.636571:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.636572:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1784949299.636574:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cd114eaa. 00000100:00000040:2.0:1784949299.636585:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949299.636586:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949299.636597:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949299.636603:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949299.636610:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.636612:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949299.636617:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949299.636620:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f495c80e. 02000000:00000001:2.0:1784949299.636622:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949299.636629:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949299.636630:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949299.636640:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949299.636647:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289687936:0@lo:400: 00000100:00000040:1.0:1784949299.636649:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949299.636660:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949299.636670:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949299.636675:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949299.636682:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949299.636684:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:2.0:1784949299.636690:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000059e5b358. 00000100:00000001:1.0:1784949299.636691:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949299.636699:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949299.636709:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd114eaa] to pc [ptlrpcd_00_02+2] req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949299.636711:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e2d054c8. 00000400:00000010:2.0:1784949299.636726:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000100:00000200:2.0:1784949299.636742:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289687936, portal 25 00000100:00000001:2.0:1784949299.636747:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949299.636749:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469564544 : -115544239987072 : ffff96e9c29ea680) 00000100:00000040:2.0:1784949299.636757:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949299.636768:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949299.636770:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949299.636779:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:1.0:1784949299.636782:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11576) 00000400:00000010:2.0:1784949299.636785:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000100:00000200:2.0:1784949299.636793:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289687936, offset 0 00000100:00000001:3.0:1784949299.636802:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949299.636804:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949299.636811:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949299.636816:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949299.636822:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949299.636829:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949299.636831:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949299.636833:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949299.636853:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949299.636857:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949299.636863:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e780 02000000:00000001:3.0:1784949299.636863:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949299.636870:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949299.636874:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949299.636879:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289688064:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949299.636891:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949299.636899:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1784949299.636900:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8b81 [2] + 0 00000400:00000010:2.0:1784949299.636945:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000000b3b2e3c. 00000400:00000200:2.0:1784949299.636954:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000a21d030b 00000400:00000200:2.0:1784949299.636967:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784949299.636967:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949299.636980:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784949299.636985:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000010:3.0:1784949299.636993:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000009f047fe3. 00000400:00000200:2.0:1784949299.636994:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a21d030b 00000400:00000010:2.0:1784949299.637000:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000a21d030b. 02000000:00000001:3.0:1784949299.637004:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.637011:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949299.637016:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000010:3.0:1784949299.637018:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000100:00000040:2.0:1784949299.637025:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000065a766c3 x1871651289687936 msgsize 224 00000400:00000010:3.0:1784949299.637030:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000100:00100000:2.0:1784949299.637044:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000200:3.0:1784949299.637045:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289688064, portal 10 00000100:00000040:2.0:1784949299.637054:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949299.637055:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949299.637060:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637804672 : -115544071746944 : ffff96e9cca5ca80) 00000100:00000040:3.0:1784949299.637075:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949299.637078:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949299.637093:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784949299.637097:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000100:00000040:3.0:1784949299.637106:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949299.637107:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000400:00000010:3.0:1784949299.637115:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000467f058f. 00000100:00000001:2.0:1784949299.637120:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949299.637123:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289688064, offset 0 00000100:00000200:2.0:1784949299.637132:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949299.637141:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949299.637158:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784949299.637165:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:2.0:1784949299.637167:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949299.637168:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949299.637179:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289687936 00000400:00000200:3.0:1784949299.637179:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784949299.637191:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949299.637193:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949299.637197:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949299.637201:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949299.637201:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949299.637212:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949299.637212:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784949299.637227:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949299.637228:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00100000:0.0:1784949299.637241:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289687936 00000400:00000200:3.0:1784949299.637244:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3863:3863:242:4294967295] 192.168.202.137@tcp LPNI seq info [2471:2471:8:4294967295] 00000020:00000001:0.0:1784949299.637251:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949299.637260:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949299.637266:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949299.637273:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784949299.637285:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:3.0:1784949299.637287:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:0.0:1784949299.637297:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949299.637308:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949299.637310:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:3.0:1784949299.637311:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949299.637317:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306239019 00000020:00000010:0.0:1784949299.637323:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000d4933c9b. 00000800:00000010:3.0:1784949299.637326:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000010:2.0:1784949299.637328:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cb04e419. 00000100:00000001:0.0:1784949299.637335:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949299.637346:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000d4933c9b. 00000100:00000001:2.0:1784949299.637347:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000800:00000200:3.0:1784949299.637352:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949299.637366:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306239019 00000020:00000010:0.0:1784949299.637367:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000800:00000200:3.0:1784949299.637378:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000010:2.0:1784949299.637382:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a2179890. 00000020:00000010:0.0:1784949299.637386:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000734334de. 00000800:00000200:3.0:1784949299.637391:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949299.637393:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:0.0:1784949299.637401:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002ffa19fa. 00000100:00000001:2.0:1784949299.637403:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306239019 00000100:00000010:2.0:1784949299.637417:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000010de4e21. 00000100:00000001:2.0:1784949299.637425:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000040:0.0:1784949299.637430:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949299.637431:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.637433:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306239019 00000100:00000001:0.0:1784949299.637442:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:2.0:1784949299.637445:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e2039f57. 00000100:00000040:3.0:1784949299.637448:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2247 usecs (3 jiffies) 00000100:00000001:0.0:1784949299.637451:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949299.637456:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949299.637460:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.637462:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306238980 00000100:00000001:2.0:1784949299.637491:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949299.637502:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.637505:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306239019 00000100:00000001:2.0:1784949299.637508:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949299.637509:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.637519:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.637519:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306238980 00000100:00000001:2.0:1784949299.637532:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949299.637532:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306239019 00000100:00000001:0.0:1784949299.637537:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949299.637538:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784949299.637542:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306238980 00000100:00000001:2.0:1784949299.637552:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949299.637555:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306239019 00000100:00000001:2.0:1784949299.637557:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949299.637559:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949299.637563:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949299.637567:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949299.637569:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306238980 00000100:00000001:0.0:1784949299.637570:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949299.637571:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949299.637579:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949299.637583:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 822 00000100:00000001:3.0:1784949299.637584:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306239019 00000100:00000040:1.0:1784949299.637593:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949299.637595:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949299.637599:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770425424 : -115539939126192 : ffff96eac2f89450) 00000100:00000001:0.0:1784949299.637601:0:342757:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.637609:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949299.637612:0:342757:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949299.637616:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000065a766c3 x1871651289687936/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:205/0 lens 224/0 e 0 to 0 dl 1784949310 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949299.637625:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.637636:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.637652:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949299.637654:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949299.637660:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:1.0:1784949299.637666:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000004204c760. 00000100:00000001:3.0:1784949299.637669:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949299.637679:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949299.637680:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949299.637684:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949299.637692:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000004204c760. 00000100:00000001:3.0:1784949299.637695:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949299.637710:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000065a766c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289687936:12345-0@lo:400: 00000100:00000200:1.0:1784949299.637721:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289687936 00000020:00000001:1.0:1784949299.637728:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949299.637736:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000200:2.0:1784949299.637737:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:1.0:1784949299.637741:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949299.637745:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000020:00000001:1.0:1784949299.637752:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:2.0:1784949299.637755:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784949299.637760:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:2.0:1784949299.637770:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:1.0:1784949299.637775:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:2.0:1784949299.637777:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000467f058f 00000020:00000001:1.0:1784949299.637786:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949299.637786:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000467f058f. 00000800:00000001:0.0:1784949299.637793:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949299.637796:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1784949299.637798:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949299.637804:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000200:2.0:1784949299.637808:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949299.637811:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949299.637816:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949299.637821:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:0.0:1784949299.637831:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949299.637832:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784949299.637835:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1784949299.637842:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000040:2.0:1784949299.637843:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949299.637844:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1784949299.637849:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000004cb3d562. 02000000:00000001:1.0:1784949299.637859:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949299.637868:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949299.637877:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:1.0:1784949299.637877:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.637880:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949299.637888:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289687936 00000100:00000001:2.0:1784949299.637889:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1784949299.637897:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949299.637959:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000065a766c3 x1871651289687936/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:205/0 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949299.637970:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e800 00010000:00000001:1.0:1784949299.637988:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949299.637992:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949299.637999:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 179792 00000100:00001000:1.0:1784949299.638002:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=21 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949299.638016:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:0.0:1784949299.638022:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949299.638023:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949299.638030:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784949299.638034:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949299.638037:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949299.638038:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949299.638041:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949299.638049:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784949299.638050:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949299.638057:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000200:0.0:1784949299.638067:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:1.0:1784949299.638067:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289687936, offset 224 00000400:00000200:1.0:1784949299.638083:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949299.638092:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949299.638099:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:1.0:1784949299.638106:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e780 00000100:00000200:0.0:1784949299.638108:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:1.0:1784949299.638118:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x958d [1] + 224 00000100:00000010:0.0:1784949299.638121:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c69adecd. 00000400:00000200:1.0:1784949299.638127:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:0.0:1784949299.638130:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000c69adecd x1871651289688064 msgsize 224 00000400:00000200:1.0:1784949299.638137:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949299.638147:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949299.638157:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949299.638160:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000040:1.0:1784949299.638178:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949299.638191:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949299.638205:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949299.638208:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949299.638218:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000800:00000001:0.0:1784949299.638220:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949299.638223:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000100:00000001:1.0:1784949299.638234:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949299.638237:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784949299.638244:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.638248:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:1.0:1784949299.638250:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:1.0:1784949299.638258:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00100000:2.0:1784949299.638259:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289688064 00000020:00000001:1.0:1784949299.638264:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949299.638265:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949299.638271:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:1.0:1784949299.638273:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:2.0:1784949299.638276:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949299.638283:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000065a766c3 x1871651289687936/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:205/0 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949299.638284:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.638286:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949299.638297:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289688064 00000100:00000001:3.0:1784949299.638297:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1784949299.638308:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000001:3.0:1784949299.638314:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:1.0:1784949299.638315:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000065a766c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289687936:12345-0@lo:400: Request processed in 656us (1283us total) trans 0 rc 0/0 00000020:00000040:2.0:1784949299.638316:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000001:3.0:1784949299.638324:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1784949299.638325:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:3.0:1784949299.638335:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e2d054c8. 00000020:00000040:2.0:1784949299.638336:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00100000:1.0:1784949299.638343:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 822 00000020:00000001:2.0:1784949299.638349:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000040:1.0:1784949299.638352:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000400:00000200:3.0:1784949299.638354:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000020:00000001:2.0:1784949299.638355:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949299.638362:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000010:3.0:1784949299.638365:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000100:00000001:2.0:1784949299.638367:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949299.638369:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949299.638378:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:3.0:1784949299.638380:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:1.0:1784949299.638381:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000734334de. 00000100:00000001:2.0:1784949299.638385:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949299.638393:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000020:00000010:1.0:1784949299.638395:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002ffa19fa. 00000100:00000200:3.0:1784949299.638398:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949299.638404:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b1cbf66f. 00000020:00000010:1.0:1784949299.638407:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000020:00000010:2.0:1784949299.638418:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085067f70. 00000020:00000040:1.0:1784949299.638421:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000010:2.0:1784949299.638431:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 02000000:00000001:1.0:1784949299.638433:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949299.638440:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000004cb3d562. 00000100:00000200:3.0:1784949299.638446:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949299.638448:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:1.0:1784949299.638449:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949299.638456:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949299.638459:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949299.638464:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949299.638495:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949299.638499:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784949299.638509:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000010:1.0:1784949299.638512:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004aad1993. 00000100:00000001:2.0:1784949299.638516:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949299.638523:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.638525:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000010:1.0:1784949299.638526:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006b6903df. 00000100:00000001:3.0:1784949299.638536:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949299.638543:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.638546:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949299.638555:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949299.638561:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949299.638565:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.638566:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949299.638572:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:3.0:1784949299.638574:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949299.638583:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 798 00000100:00001000:3.0:1784949299.638587:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=21 v=5 (5 5 5 5) 00000100:00000040:2.0:1784949299.638589:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949299.638597:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469569024 : -115544239982592 : ffff96e9c29eb800) 00000100:00001000:3.0:1784949299.638600:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=21 v=5 (1 1 1 1) 00000100:00000040:2.0:1784949299.638609:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000c69adecd x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:205/0 lens 224/0 e 0 to 0 dl 1784949310 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949299.638610:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949299.638617:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949299.638626:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949299.638632:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.638643:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949299.638657:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949299.638660:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949299.638667:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:2.0:1784949299.638675:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949299.638685:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000100:00100000:2.0:1784949299.638702:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000c69adecd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289688064:12345-192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949299.638705:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1784949299.638713:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289688064 00000100:00000001:3.0:1784949299.638715:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1784949299.638721:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949299.638722:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949299.638732:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:3.0:1784949299.638734:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949299.638739:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949299.638750:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949299.638755:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00100000:3.0:1784949299.638761:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289687936:0@lo:400: 00000020:00000001:2.0:1784949299.638764:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949299.638771:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.638774:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949299.638777:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949299.638780:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949299.638781:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:3.0:1784949299.638787:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ca646268 x1871651289687936/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949299.638788:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949299.638798:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949299.638804:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949299.638810:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949299.638811:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:3.0:1784949299.638816:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949299.638819:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007a276b4a. 02000000:00000010:3.0:1784949299.638821:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000059e5b358. 02000000:00000001:2.0:1784949299.638828:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949299.638832:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1784949299.638834:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949299.638838:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:2.0:1784949299.638840:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949299.638843:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00010000:00000040:2.0:1784949299.638846:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289688064 00000020:00000001:3.0:1784949299.638850:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000001:2.0:1784949299.638853:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000010:3.0:1784949299.638855:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d104522b. 02000000:00000001:3.0:1784949299.638862:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00010000:00000200:2.0:1784949299.638863:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000c69adecd x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:205/0 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949299.638866:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949299.638872:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ca646268. 00000100:00000001:3.0:1784949299.638882:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949299.638888:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949299.638889:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949299.638896:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949299.638897:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949299.638937:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=16 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949299.638943:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949299.638944:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1784949299.638949:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949299.638951:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.638959:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:3.0:1784949299.638961:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949299.638968:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784949299.638969:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949299.638974:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949299.638980:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949299.638984:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949299.638989:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000100:00000200:2.0:1784949299.638994:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289688064, offset 224 00000400:00000200:2.0:1784949299.639000:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949299.639009:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949299.639015:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3864:3864:242:4294967295] 192.168.202.137@tcp LPNI seq info [2472:2472:8:4294967295] 00000400:00000200:2.0:1784949299.639027:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949299.639032:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949299.639037:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000adf08f9b. 00000800:00000200:2.0:1784949299.639042:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949299.639049:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949299.639053:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000adf08f9b type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949299.639072:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949299.639076:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949299.639079:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949299.639081:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949299.639083:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949299.639088:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c69adecd x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:205/0 lens 224/224 e 0 to 0 dl 1784949310 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949299.639099:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000c69adecd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289688064:12345-192.168.202.137@tcp:400: Request processed in 438us (968us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949299.639107:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 798 00000100:00000040:2.0:1784949299.639111:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949299.639114:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949299.639116:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949299.639122:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085067f70. 00000020:00000010:2.0:1784949299.639128:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949299.639133:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b1cbf66f. 00000020:00000040:2.0:1784949299.639138:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949299.639141:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949299.639291:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949299.639297:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949299.639300:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000adf08f9b. 00000800:00000001:2.0:1784949299.639310:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949299.639313:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949299.639319:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949299.639325:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949299.639334:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949299.639345:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949299.639351:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000010:0.0:1784949299.639359:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000400:00000200:2.0:1784949299.639360:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e800 00000100:00000001:0.0:1784949299.639373:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:2.0:1784949299.639375:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9595 [1] + 224 00000100:00000001:0.0:1784949299.639382:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949299.639387:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949299.639401:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949299.639404:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949299.639409:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949299.639416:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949299.639422:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949299.639434:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949299.639455:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949299.639462:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949299.639465:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.639528:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949299.639533:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949299.639539:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949299.639540:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949299.639543:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000400:00000200:3.0:1784949299.639551:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000400:00000010:3.0:1784949299.639554:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000100:00000001:3.0:1784949299.639558:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949299.639564:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949299.639575:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949299.639584:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949299.639585:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.639588:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949299.639590:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949299.639591:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949299.639593:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949299.639595:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949299.639601:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=16 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949299.639605:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=16 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949299.639608:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949299.639609:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949299.639612:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949299.639615:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949299.639624:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.639627:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949299.639629:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949299.639631:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949299.639633:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949299.639636:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949299.639644:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949299.639647:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949299.639648:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949299.639652:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949299.639660:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000cd114eaa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289688064:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949299.639665:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.639668:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949299.639670:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000cd114eaa x1871651289688064/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949299.639678:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949299.639680:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949299.639682:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000009f047fe3. 02000000:00000001:3.0:1784949299.639685:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949299.639686:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949299.639688:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949299.639690:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949299.639691:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f495c80e. 02000000:00000001:3.0:1784949299.639695:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949299.639697:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949299.639698:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000cd114eaa. 00000100:00000001:3.0:1784949299.639701:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949299.639702:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.639704:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949299.639709:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949299.639710:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949299.639713:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949299.639715:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949299.680006:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306239019, at 4306239020 00000100:00000010:1.0:1784949299.680022:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a2179890. 00000100:00000040:0.0:1784949299.680030:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306239019, at 4306239020 00000100:00000040:2.0:1784949299.680035:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306239019, at 4306239020 00000100:00000010:0.0:1784949299.680045:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cb04e419. 00000100:00000040:3.0:1784949299.680049:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306239019, at 4306239020 00000100:00000010:2.0:1784949299.680051:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000010de4e21. 00000100:00000010:3.0:1784949299.680067:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e2039f57. 00010000:00000001:2.0:1784949300.468061:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949300.468072:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949300.468079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949300.468081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949300.468162:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949300.468169:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949300.468174:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949300.660038:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949300.660057:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949300.660062:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949300.660071:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949300.660075:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949300.660082:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949300.660087:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949301.492029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949301.492038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949301.492046:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949301.492050:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949301.492054:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949301.492057:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949301.492061:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949301.492064:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949301.492066:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949301.492069:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949301.492177:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949301.492186:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949301.492193:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949301.684002:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949301.684018:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949301.684021:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949301.684027:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949301.684032:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949301.684036:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949301.684040:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949302.515977:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949302.515984:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949302.515989:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949302.515991:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949302.516052:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949302.516058:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949302.516062:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949302.707978:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949302.707995:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949302.707998:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949302.708004:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949302.708007:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949302.708012:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949302.708015:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949303.540074:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949303.540083:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949303.540091:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949303.540093:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949303.540190:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949303.540197:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949303.540202:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949303.732036:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949303.732055:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949303.732058:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949303.732066:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949303.732069:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949303.732074:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949303.732078:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949304.564040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949304.564051:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949304.564059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949304.564061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949304.564187:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949304.564194:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949304.564200:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949304.756035:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949304.756053:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949304.756057:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949304.756066:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949304.756069:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949304.756074:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.756077:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949304.756167:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949304.756177:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949304.756187:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f84ecbdd. 00000020:00000040:0.0:1784949304.756196:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949304.756203:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949304.756208:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949304.756214:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949304.756226:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 00000100:00000001:0.0:1784949304.756237:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949304.756250:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949304.756266:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949304.756270:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949304.756278:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f84ecbdd] to pc [ptlrpcd_00_03+3] req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949304.756319:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949304.756324:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949304.756330:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949304.756335:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949304.756340:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949304.756346:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000021c72e86. 00000020:00000040:0.0:1784949304.756351:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949304.756354:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949304.756356:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949304.756359:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949304.756360:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.756369:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949304.756373:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000062b4a223. 00000100:00100000:3.0:1784949304.756374:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:2.0:1784949304.756376:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949304.756384:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.756391:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949304.756399:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949304.756409:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949304.756413:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949304.756417:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949304.756419:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.756451:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949304.756457:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949304.756465:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949304.756470:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021c72e86] to pc [ptlrpcd_00_00+0] req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949304.756482:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949304.756506:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949304.756511:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.756512:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949304.756526:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949304.756527:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11581) 00000100:00000001:2.0:1784949304.756534:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949304.756537:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949304.756543:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949304.756547:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.756549:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1784949304.756564:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f84ecbdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289688192:0@lo:400: 00000100:00000040:2.0:1784949304.756565:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949304.756573:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949304.756581:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.756589:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949304.756602:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949304.756606:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949304.756606:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949304.756617:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949304.756618:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949304.756623:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949304.756628:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1784949304.756631:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949304.756640:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000021c72e86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289688320:192.168.202.137@tcp:400: 02000000:00000010:3.0:1784949304.756642:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000378c7327. 00000100:00000001:2.0:1784949304.756651:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949304.756654:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949304.756658:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949304.756665:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949304.756669:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:3.0:1784949304.756672:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 02000000:00000010:2.0:1784949304.756676:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000b1cbf66f. 02000000:00000001:2.0:1784949304.756683:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949304.756693:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000400:00000010:2.0:1784949304.756694:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f4e53d4f. 00000400:00000010:2.0:1784949304.756710:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000100:00000200:3.0:1784949304.756715:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289688192, portal 25 00000100:00000200:2.0:1784949304.756718:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289688320, portal 10 00000100:00000001:2.0:1784949304.756728:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949304.756728:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949304.756733:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469563648 : -115544239987968 : ffff96e9c29ea300) 00000100:00000001:3.0:1784949304.756737:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469560064 : -115544239991552 : ffff96e9c29e9500) 00000100:00000040:2.0:1784949304.756747:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949304.756757:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949304.756766:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949304.756779:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949304.756789:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000100:00000001:3.0:1784949304.756799:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000200:2.0:1784949304.756800:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289688320, offset 0 00000100:00000040:3.0:1784949304.756817:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:2.0:1784949304.756818:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949304.756832:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000200:3.0:1784949304.756843:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289688192, offset 0 00000400:00000200:2.0:1784949304.756847:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949304.756859:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949304.756866:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:2.0:1784949304.756868:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949304.756880:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949304.756884:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:2.0:1784949304.756891:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3865:3865:242:4294967295] 192.168.202.137@tcp LPNI seq info [2473:2473:8:4294967295] 00000400:00000200:2.0:1784949304.756944:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949304.756953:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949304.756961:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 00000400:00000200:3.0:1784949304.756973:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:2.0:1784949304.756982:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949304.756995:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784949304.757001:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1784949304.757006:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e880 00000400:00000200:3.0:1784949304.757025:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8ba5 [2] + 0 00000100:00000001:2.0:1784949304.757030:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949304.757040:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002f6a97f7. 00000100:00000040:2.0:1784949304.757041:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2250 usecs (3 jiffies) 00000100:00000001:2.0:1784949304.757053:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306244100 00000400:00000200:3.0:1784949304.757056:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000043dfa704 00000100:00000010:2.0:1784949304.757060:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000010de4e21. 00000400:00000200:3.0:1784949304.757074:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949304.757075:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000400:00000200:3.0:1784949304.757091:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949304.757094:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306244100 00000100:00000010:2.0:1784949304.757101:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a3d0cc2f. 00000400:00000200:3.0:1784949304.757103:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000043dfa704 00000100:00000001:2.0:1784949304.757107:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000400:00000010:3.0:1784949304.757114:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000043dfa704. 00000100:00000001:2.0:1784949304.757119:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306244100 00000100:00000001:3.0:1784949304.757126:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000010:2.0:1784949304.757131:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000098689e07. 00000100:00000200:3.0:1784949304.757134:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:2.0:1784949304.757143:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000040:3.0:1784949304.757147:0:337121:0:(events.c:337:request_in_callback()) incoming req@000000009c11a764 x1871651289688192 msgsize 224 00000100:00000001:2.0:1784949304.757160:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306244100 00000100:00100000:3.0:1784949304.757170:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000010:2.0:1784949304.757175:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008a834f68. 00000100:00000040:3.0:1784949304.757178:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949304.757189:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949304.757203:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949304.757209:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949304.757210:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.757221:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949304.757227:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949304.757227:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949304.757230:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000800:00000010:1.0:1784949304.757239:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000400:00000010:3.0:1784949304.757241:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:2.0:1784949304.757242:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949304.757250:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:1.0:1784949304.757254:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949304.757256:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949304.757257:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949304.757264:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949304.757270:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949304.757272:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949304.757274:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949304.757283:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000100:00000001:2.0:1784949304.757283:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949304.757294:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000100:00000001:1.0:1784949304.757306:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949304.757315:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1784949304.757318:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949304.757330:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949304.757340:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949304.757351:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949304.757363:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949304.757365:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949304.757372:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289688192 00000800:00000001:0.0:1784949304.757375:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949304.757378:0:337121:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784949304.757380:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949304.757385:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.757388:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949304.757391:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949304.757393:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949304.757397:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.757398:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949304.757401:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949304.757409:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949304.757414:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306244139 00000100:00100000:2.0:1784949304.757415:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289688192 00000800:00000001:0.0:1784949304.757419:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949304.757455:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949304.757461:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949304.757464:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306244100 00000400:00000200:0.0:1784949304.757471:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:2.0:1784949304.757479:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949304.757489:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784949304.757497:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949304.757497:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306244139 00000020:00000001:2.0:1784949304.757506:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784949304.757508:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643e900 00000100:00000001:3.0:1784949304.757515:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306244100 00000100:00000001:2.0:1784949304.757519:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949304.757527:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 180016 00000020:00000010:2.0:1784949304.757531:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000008aed2f1. 00000100:00000001:3.0:1784949304.757534:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306244139 00000800:00000001:0.0:1784949304.757537:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949304.757541:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949304.757548:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000008aed2f1. 00000800:00000001:0.0:1784949304.757552:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.757553:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306244100 00000020:00000010:2.0:1784949304.757560:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000059e5b358. 00000400:00000200:0.0:1784949304.757561:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949304.757567:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306244139 00000020:00000010:2.0:1784949304.757575:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085067f70. 00000400:00000200:0.0:1784949304.757582:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949304.757584:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306244100 00000020:00000010:2.0:1784949304.757588:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:0.0:1784949304.757594:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949304.757600:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784949304.757604:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1784949304.757608:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000050bfb6db. 00000100:00000040:2.0:1784949304.757609:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000040:0.0:1784949304.757621:0:337113:0:(events.c:337:request_in_callback()) incoming req@0000000050bfb6db x1871651289688320 msgsize 224 00000100:00000001:2.0:1784949304.757622:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949304.757622:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.757629:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949304.757634:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.757637:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949304.757643:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:3.0:1784949304.757650:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.757656:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949304.757668:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949304.757676:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949304.757682:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949304.757691:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949304.757697:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949304.757703:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949304.757710:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 823 00000100:00000040:2.0:1784949304.757716:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949304.757720:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628533770428496 : -115539939123120 : ffff96eac2f8a050) 00000100:00000040:2.0:1784949304.757728:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009c11a764 x1871651289688192/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:210/0 lens 224/0 e 0 to 0 dl 1784949315 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949304.757742:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949304.757747:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000008aed2f1. 00000100:00000001:2.0:1784949304.757751:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949304.757754:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000008aed2f1. 00000100:00100000:2.0:1784949304.757763:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009c11a764 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289688192:12345-0@lo:400: 00000100:00000200:2.0:1784949304.757769:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289688192 00000020:00000001:2.0:1784949304.757773:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949304.757779:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949304.757782:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949304.757784:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:2.0:1784949304.757786:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00100000:1.0:1784949304.757794:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289688320 00000020:00000001:2.0:1784949304.757794:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 02000000:00000001:1.0:1784949304.757803:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:2.0:1784949304.757805:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949304.757810:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:2.0:1784949304.757812:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949304.757818:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949304.757819:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949304.757828:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:1.0:1784949304.757830:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949304.757833:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949304.757836:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289688320 00000020:00000001:2.0:1784949304.757839:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000001:1.0:1784949304.757843:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949304.757845:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000040:1.0:1784949304.757848:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000001:2.0:1784949304.757852:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000001:1.0:1784949304.757854:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949304.757860:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 02000000:00000010:2.0:1784949304.757862:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007672855d. 00000020:00000001:1.0:1784949304.757866:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:2.0:1784949304.757869:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949304.757874:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949304.757876:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949304.757884:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949304.757885:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000040:2.0:1784949304.757891:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289688192 00000020:00000010:1.0:1784949304.757895:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00010000:00000001:2.0:1784949304.757896:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949304.757944:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000200:2.0:1784949304.757950:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009c11a764 x1871651289688192/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:210/0 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949304.757952:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000020:00000010:1.0:1784949304.757967:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000bbdbdf4. 00010000:00000001:2.0:1784949304.757969:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949304.757976:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949304.757979:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a39e81d9. 00000100:00001000:2.0:1784949304.757982:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=26 v=5 (1 1 1 1) 00000020:00000010:1.0:1784949304.757989:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:2.0:1784949304.757992:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949304.758001:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949304.758006:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000040:1.0:1784949304.758009:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:2.0:1784949304.758010:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949304.758015:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949304.758017:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949304.758021:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949304.758023:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949304.758029:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949304.758031:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949304.758041:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000100:00000001:1.0:1784949304.758042:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949304.758050:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:2.0:1784949304.758052:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289688192, offset 224 00000100:00000001:1.0:1784949304.758054:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949304.758065:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949304.758074:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949304.758077:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949304.758092:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e880 00000100:00000001:1.0:1784949304.758098:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784949304.758104:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x95a9 [1] + 224 00000100:00000001:1.0:1784949304.758105:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949304.758119:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 799 00000400:00000200:2.0:1784949304.758121:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1784949304.758123:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949304.758131:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469557376 : -115544239994240 : ffff96e9c29e8a80) 00000400:00000200:2.0:1784949304.758140:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:1.0:1784949304.758146:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000050bfb6db x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:210/0 lens 224/0 e 0 to 0 dl 1784949315 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949304.758157:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784949304.758161:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949304.758168:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000004262fc5. 00000100:00000200:2.0:1784949304.758173:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949304.758174:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949304.758183:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000004262fc5. 00000100:00100000:1.0:1784949304.758200:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000050bfb6db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289688320:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949304.758216:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289688320 00000020:00000001:1.0:1784949304.758223:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949304.758227:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000040:2.0:1784949304.758228:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949304.758233:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949304.758241:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949304.758248:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949304.758256:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949304.758259:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949304.758261:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949304.758263:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949304.758265:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949304.758267:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949304.758270:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949304.758273:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949304.758278:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949304.758282:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000068d8ca1f. 00000400:00000200:2.0:1784949304.758283:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 02000000:00000001:1.0:1784949304.758287:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949304.758289:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:1.0:1784949304.758293:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.758297:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:1.0:1784949304.758298:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.758303:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00010000:00000040:1.0:1784949304.758305:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289688320 00010000:00000001:1.0:1784949304.758309:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949304.758316:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000050bfb6db x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:210/0 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949304.758327:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949304.758328:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1784949304.758330:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.758338:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00001000:1.0:1784949304.758339:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=21 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949304.758344:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949304.758349:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:3.0:1784949304.758354:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784949304.758355:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000040:1.0:1784949304.758359:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000400:00000001:3.0:1784949304.758365:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:2.0:1784949304.758366:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949304.758368:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:2.0:1784949304.758376:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949304.758377:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:3.0:1784949304.758377:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 02000000:00000001:1.0:1784949304.758384:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949304.758387:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949304.758392:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949304.758397:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000100:00000040:2.0:1784949304.758401:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009c11a764 x1871651289688192/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:210/0 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949304.758406:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949304.758409:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000400:00000010:1.0:1784949304.758415:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000001:3.0:1784949304.758451:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949304.758455:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289688320, offset 224 00000100:00000200:3.0:1784949304.758469:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949304.758478:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009c11a764 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289688192:12345-0@lo:400: Request processed in 725us (1320us total) trans 0 rc 0/0 00000400:00000200:1.0:1784949304.758479:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949304.758503:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00100000:2.0:1784949304.758520:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 823 00000400:00000200:1.0:1784949304.758530:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3866:3866:242:4294967295] 192.168.202.137@tcp LPNI seq info [2474:2474:8:4294967295] 00000100:00000040:2.0:1784949304.758533:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949304.758543:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:3.0:1784949304.758544:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949304.758551:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949304.758562:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085067f70. 00000400:00000200:1.0:1784949304.758570:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:2.0:1784949304.758575:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949304.758586:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000059e5b358. 00000800:00000200:1.0:1784949304.758599:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000040:2.0:1784949304.758599:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949304.758606:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000010:1.0:1784949304.758612:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000005678171. 02000000:00000001:2.0:1784949304.758613:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949304.758615:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784949304.758621:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007672855d. 00000100:00000001:3.0:1784949304.758627:0:337121:0:(client.c:1580:after_reply()) Process entered 00000800:00000200:1.0:1784949304.758632:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:2.0:1784949304.758633:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:3.0:1784949304.758636:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949304.758639:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.758645:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:1.0:1784949304.758648:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000200:2.0:1784949304.758649:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949304.758654:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949304.758659:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000005678171 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:2.0:1784949304.758663:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a0670fc. 02000000:00000001:3.0:1784949304.758664:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949304.758675:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d5e12ff0. 00000100:00001000:3.0:1784949304.758680:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=26 v=5 (5 5 5 5) 00000100:00000001:1.0:1784949304.758683:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949304.758696:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00001000:3.0:1784949304.758698:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=26 v=5 (1 1 1 1) 00010000:00000001:1.0:1784949304.758701:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949304.758707:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.758712:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:1.0:1784949304.758714:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784949304.758721:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949304.758723:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000050bfb6db x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:210/0 lens 224/224 e 0 to 0 dl 1784949315 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949304.758733:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949304.758742:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949304.758745:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000050bfb6db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289688320:12345-192.168.202.137@tcp:400: Request processed in 581us (1121us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949304.758754:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949304.758763:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 799 00000100:00000040:1.0:1784949304.758769:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000040:3.0:1784949304.758774:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949304.758775:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949304.758782:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949304.758788:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a39e81d9. 00000020:00000010:1.0:1784949304.758798:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949304.758808:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949304.758810:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000bbdbdf4. 00000020:00000040:1.0:1784949304.758819:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949304.758822:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949304.758825:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.758829:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949304.758841:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949304.758873:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f84ecbdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289688192:0@lo:400: 00000800:00000200:0.0:1784949304.758881:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949304.758884:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949304.758894:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949304.758895:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000005678171. 00000100:00000001:3.0:1784949304.758895:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:2.0:1784949304.758943:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949304.758949:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949304.758952:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f84ecbdd x1871651289688192/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:2.0:1784949304.758957:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949304.758959:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949304.758985:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:2.0:1784949304.758987:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949304.758993:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784949304.758994:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949304.759002:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000378c7327. 00000400:00000200:0.0:1784949304.759007:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000200:2.0:1784949304.759011:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643e900 02000000:00000001:3.0:1784949304.759014:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000010:0.0:1784949304.759015:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000020:00000001:3.0:1784949304.759022:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949304.759026:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949304.759028:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:3.0:1784949304.759031:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:0.0:1784949304.759035:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 02000000:00000010:3.0:1784949304.759037:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 00000400:00000200:2.0:1784949304.759045:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x95ad [1] + 224 02000000:00000001:3.0:1784949304.759051:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949304.759056:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000001:2.0:1784949304.759057:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000010:3.0:1784949304.759059:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f84ecbdd. 00000100:00000001:3.0:1784949304.759068:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949304.759071:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.759074:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949304.759077:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.759082:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1784949304.759084:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949304.759088:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949304.759098:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949304.759099:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949304.759105:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949304.759109:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949304.759120:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949304.759139:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949304.759168:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949304.759181:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949304.759188:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.759199:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949304.759205:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949304.759213:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949304.759218:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949304.759222:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f4e53d4f. 00000400:00000200:3.0:1784949304.759232:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:3.0:1784949304.759235:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000100:00000001:3.0:1784949304.759241:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949304.759249:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949304.759267:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949304.759282:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949304.759285:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.759288:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949304.759293:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949304.759294:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949304.759296:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949304.759299:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949304.759305:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=21 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949304.759309:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=21 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949304.759312:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949304.759314:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949304.759316:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949304.759321:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949304.759330:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.759334:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949304.759336:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949304.759339:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949304.759340:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949304.759344:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949304.759351:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949304.759353:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949304.759355:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949304.759358:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949304.759366:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000021c72e86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289688320:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949304.759372:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.759374:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949304.759377:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000021c72e86 x1871651289688320/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949304.759393:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949304.759395:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949304.759397:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000b1cbf66f. 02000000:00000001:3.0:1784949304.759400:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949304.759402:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949304.759404:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949304.759406:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949304.759408:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000062b4a223. 02000000:00000001:3.0:1784949304.759411:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949304.759412:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949304.759413:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000021c72e86. 00000100:00000001:3.0:1784949304.759418:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949304.759419:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.759421:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949304.759449:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949304.759453:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949304.759456:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949304.759458:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949304.800010:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306244139, at 4306244140 00000100:00000010:0.0:1784949304.800024:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000010de4e21. 00000100:00000040:1.0:1784949304.800037:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306244139, at 4306244140 00000100:00000040:2.0:1784949304.800051:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306244139, at 4306244140 00000100:00000010:1.0:1784949304.800052:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a3d0cc2f. 00000100:00000040:3.0:1784949304.800058:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306244139, at 4306244140 00000100:00000010:2.0:1784949304.800069:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000098689e07. 00000100:00000010:3.0:1784949304.800073:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008a834f68. 00010000:00000001:2.0:1784949305.588045:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949305.588057:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949305.588069:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949305.588072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949305.588158:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949305.588165:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949305.588170:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949305.780048:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949305.780081:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949305.780085:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949305.780095:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949305.780100:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949305.780107:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949305.780112:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949306.612026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949306.612035:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949306.612043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949306.612045:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949306.612118:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949306.612125:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949306.612130:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949306.804009:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949306.804026:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949306.804028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949306.804034:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949306.804038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949306.804042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949306.804047:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949307.636049:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949307.636059:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949307.636067:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949307.636069:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949307.636208:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949307.636214:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949307.636218:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949307.828032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949307.828047:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949307.828049:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949307.828056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949307.828059:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949307.828064:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949307.828067:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949308.660017:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949308.660026:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949308.660033:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949308.660034:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949308.660147:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949308.660154:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949308.660158:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949308.852046:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949308.852066:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949308.852069:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949308.852077:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949308.852081:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949308.852086:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949308.852090:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949309.683983:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949309.683989:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949309.683996:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949309.683997:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949309.684042:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949309.684045:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949309.684048:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949309.876004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949309.876015:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949309.876018:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949309.876022:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949309.876025:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949309.876028:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949309.876031:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949309.876072:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949309.876081:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949309.876090:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000029a32fda. 00000020:00000040:1.0:1784949309.876099:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949309.876107:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949309.876111:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949309.876119:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949309.876130:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000005678171. 00000100:00000001:1.0:1784949309.876140:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949309.876153:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949309.876169:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949309.876173:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949309.876180:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029a32fda] to pc [ptlrpcd_00_01+1] req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949309.876213:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1784949309.876218:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949309.876224:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949309.876227:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949309.876232:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949309.876236:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005126bdfa. 00000020:00000040:1.0:1784949309.876239:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949309.876241:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949309.876242:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949309.876249:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949309.876249:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949309.876254:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784949309.876255:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949309.876262:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f495c80e. 00000100:00000040:3.0:1784949309.876263:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949309.876271:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949309.876281:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1784949309.876286:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949309.876286:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949309.876311:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949309.876317:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949309.876327:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.876329:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1784949309.876334:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289688448:0@lo:400: 00000100:00000001:1.0:1784949309.876336:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949309.876348:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949309.876354:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005126bdfa] to pc [ptlrpcd_00_02+2] req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949309.876356:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949309.876361:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949309.876368:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949309.876377:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 02000000:00000001:3.0:1784949309.876453:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.876460:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949309.876462:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000100:00000040:1.0:1784949309.876469:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11586) 00000400:00000010:3.0:1784949309.876473:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000100:00000001:2.0:1784949309.876480:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1784949309.876483:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289688448, portal 25 00000100:00000001:2.0:1784949309.876485:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949309.876488:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949309.876489:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949309.876492:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637809152 : -115544071742464 : ffff96e9cca5dc00) 00000100:00000040:2.0:1784949309.876499:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949309.876502:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949309.876508:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949309.876515:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949309.876546:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949309.876554:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949309.876558:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000200:3.0:1784949309.876562:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289688448, offset 0 00000100:00000001:2.0:1784949309.876571:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1784949309.876573:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949309.876576:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949309.876581:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949309.876583:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949309.876586:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949309.876592:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289688576:192.168.202.137@tcp:400: 00000400:00000200:3.0:1784949309.876595:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949309.876601:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949309.876603:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949309.876607:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949309.876609:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949309.876614:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949309.876616:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643e980 02000000:00000001:2.0:1784949309.876617:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949309.876624:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000059e5b358. 00000400:00000200:3.0:1784949309.876629:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8bc9 [2] + 0 02000000:00000001:2.0:1784949309.876631:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949309.876639:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000f301fed7. 00000400:00000010:2.0:1784949309.876642:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bcdfbf21. 00000400:00000200:3.0:1784949309.876645:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000feb5569e 00000400:00000200:3.0:1784949309.876655:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1784949309.876656:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000400:00000200:3.0:1784949309.876663:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:2.0:1784949309.876665:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289688576, portal 10 00000400:00000200:3.0:1784949309.876669:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000feb5569e 00000100:00000001:2.0:1784949309.876670:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:3.0:1784949309.876673:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000feb5569e. 00000100:00000001:2.0:1784949309.876674:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637815424 : -115544071736192 : ffff96e9cca5f480) 00000100:00000001:3.0:1784949309.876681:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949309.876684:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949309.876686:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949309.876689:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000001e461ac8 x1871651289688448 msgsize 224 00000100:00100000:3.0:1784949309.876705:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784949309.876711:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949309.876714:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949309.876720:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949309.876724:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000100:00000200:2.0:1784949309.876728:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289688576, offset 0 00000100:00000001:3.0:1784949309.876733:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949309.876739:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949309.876745:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949309.876748:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:3.0:1784949309.876753:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949309.876756:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000200:3.0:1784949309.876759:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949309.876770:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784949309.876774:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949309.876779:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949309.876780:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949309.876790:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784949309.876794:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949309.876798:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:3.0:1784949309.876800:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949309.876805:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:2.0:1784949309.876809:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3867:3867:242:4294967295] 192.168.202.137@tcp LPNI seq info [2475:2475:8:4294967295] 00000100:00100000:1.0:1784949309.876811:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289688448 02000000:00000001:1.0:1784949309.876817:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949309.876820:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949309.876823:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949309.876830:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949309.876837:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:1.0:1784949309.876838:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289688448 00000020:00000001:1.0:1784949309.876844:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949309.876849:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949309.876853:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949309.876857:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949309.876858:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000040:1.0:1784949309.876859:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949309.876863:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306249258 00000020:00000001:1.0:1784949309.876868:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000010:2.0:1784949309.876873:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000061dbde97. 00000020:00000001:1.0:1784949309.876874:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784949309.876874:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:1.0:1784949309.876885:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949309.876889:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000010:1.0:1784949309.876892:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000800:00000200:2.0:1784949309.876897:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949309.876899:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949309.876901:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306249258 00000100:00000010:3.0:1784949309.876931:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:3.0:1784949309.876936:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:1.0:1784949309.876943:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000001:3.0:1784949309.876944:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306249259 00000100:00000010:3.0:1784949309.876951:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000020:00000010:1.0:1784949309.876953:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000bbdbdf4. 00000800:00000200:2.0:1784949309.876956:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784949309.876958:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:1.0:1784949309.876964:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000030e14bca. 00000800:00000200:2.0:1784949309.876967:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000061dbde97 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949309.876967:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306249259 00000020:00000010:1.0:1784949309.876973:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000010:3.0:1784949309.876976:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00000001:3.0:1784949309.876982:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784949309.876988:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949309.876990:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949309.876992:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949309.876994:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.876995:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949309.876997:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.877000:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949309.877003:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949309.877004:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2251 usecs (3 jiffies) 00000100:00000001:1.0:1784949309.877005:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949309.877011:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949309.877014:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306249220 00000100:00000001:3.0:1784949309.877015:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949309.877019:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949309.877024:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306249258 00000100:00000001:3.0:1784949309.877025:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949309.877029:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949309.877030:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306249220 00000100:00000001:3.0:1784949309.877033:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.877035:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949309.877035:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306249258 00000100:00000001:3.0:1784949309.877039:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949309.877043:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306249220 00000100:00000001:2.0:1784949309.877048:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306249259 00000100:00000001:2.0:1784949309.877050:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306249220 00000100:00000001:2.0:1784949309.877053:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306249259 00000100:00000001:2.0:1784949309.877055:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949309.877057:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949309.877058:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.877062:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949309.877064:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949309.877068:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949309.877073:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 824 00000100:00000001:2.0:1784949309.877078:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1784949309.877079:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949309.877082:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949309.877084:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180528208 : -115539529023408 : ffff96eadb6a4050) 00000100:00000001:2.0:1784949309.877090:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949309.877094:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001e461ac8 x1871651289688448/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:215/0 lens 224/0 e 0 to 0 dl 1784949320 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949309.877095:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949309.877101:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949309.877102:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949309.877108:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949309.877108:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949309.877111:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000061dbde97. 00000400:00000200:0.0:1784949309.877121:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949309.877122:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000001:3.0:1784949309.877125:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949309.877127:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000800:00000001:3.0:1784949309.877130:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949309.877136:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949309.877137:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949309.877138:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949309.877144:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000400:00000200:0.0:1784949309.877149:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000100:00100000:1.0:1784949309.877156:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001e461ac8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289688448:12345-0@lo:400: 00000400:00000010:0.0:1784949309.877157:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000400:00000200:3.0:1784949309.877158:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949309.877167:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949309.877170:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289688448 00000020:00000001:1.0:1784949309.877177:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:3.0:1784949309.877177:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ea00 00000100:00000200:0.0:1784949309.877178:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949309.877184:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949309.877190:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949309.877196:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 180240 00000020:00000001:1.0:1784949309.877197:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949309.877203:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:0.0:1784949309.877207:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1784949309.877212:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000001:3.0:1784949309.877214:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949309.877217:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949309.877218:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949309.877222:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949309.877226:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000400:00000200:3.0:1784949309.877230:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1784949309.877232:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949309.877238:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949309.877240:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949309.877244:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:3.0:1784949309.877245:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949309.877246:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949309.877251:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949309.877255:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949309.877260:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009e9a21e3. 00000100:00000001:3.0:1784949309.877260:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949309.877265:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000001:1.0:1784949309.877266:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.877272:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949309.877273:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006e540bde. 00000020:00000001:1.0:1784949309.877278:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949309.877297:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000006e540bde x1871651289688576 msgsize 224 00010000:00000040:1.0:1784949309.877304:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289688448 00010000:00000001:1.0:1784949309.877312:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00100000:3.0:1784949309.877314:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00010000:00000200:1.0:1784949309.877321:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001e461ac8 x1871651289688448/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:215/0 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949309.877332:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949309.877335:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949309.877338:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00001000:1.0:1784949309.877340:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=31 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949309.877349:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:3.0:1784949309.877354:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949309.877372:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949309.877377:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:3.0:1784949309.877379:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949309.877380:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949309.877402:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289688576 02000000:00000001:0.0:1784949309.877405:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:1.0:1784949309.877406:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:0.0:1784949309.877408:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949309.877413:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949309.877413:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949309.877422:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.877423:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949309.877426:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289688576 00000020:00000001:0.0:1784949309.877430:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:1.0:1784949309.877434:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000040:0.0:1784949309.877435:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949309.877440:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949309.877444:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000010:1.0:1784949309.877444:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000020:00000001:0.0:1784949309.877453:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:1.0:1784949309.877455:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289688448, offset 224 00000020:00000001:0.0:1784949309.877456:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949309.877463:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949309.877468:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000400:00000200:1.0:1784949309.877472:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949309.877474:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949309.877479:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000020:00000010:0.0:1784949309.877485:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000208850ab. 00000400:00000200:1.0:1784949309.877486:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:0.0:1784949309.877494:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000002fe80d84. 00000020:00000010:0.0:1784949309.877502:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000027aee7d. 00000400:00000200:1.0:1784949309.877502:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643e980 00000400:00000200:1.0:1784949309.877513:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x95c5 [1] + 224 00000100:00000040:0.0:1784949309.877515:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949309.877519:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:1.0:1784949309.877520:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949309.877522:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949309.877527:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949309.877528:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949309.877538:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949309.877539:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949309.877545:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:1.0:1784949309.877548:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949309.877550:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949309.877565:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949309.877582:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949309.877585:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949309.877590:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949309.877601:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 800 00000100:00000001:2.0:1784949309.877602:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949309.877606:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949309.877607:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949309.877611:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554957056 : -115544154594560 : ffff96e9c7b5a300) 00000100:00000001:2.0:1784949309.877613:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.877615:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949309.877620:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949309.877621:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006e540bde x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:215/0 lens 224/0 e 0 to 0 dl 1784949320 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949309.877629:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000100:00000001:0.0:1784949309.877632:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949309.877634:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784949309.877635:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000020:00000010:0.0:1784949309.877637:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e00dc9ba. 00000100:00000001:3.0:1784949309.877639:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949309.877644:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949309.877645:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949309.877649:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:3.0:1784949309.877649:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:0.0:1784949309.877650:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e00dc9ba. 00000400:00000001:3.0:1784949309.877672:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784949309.877679:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949309.877679:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000019827e71. 00000100:00100000:0.0:1784949309.877681:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006e540bde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289688576:12345-192.168.202.137@tcp:400: 00000100:00000040:1.0:1784949309.877695:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000200:0.0:1784949309.877696:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289688576 00000400:00000200:3.0:1784949309.877697:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000020:00000001:0.0:1784949309.877705:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:3.0:1784949309.877706:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00010000:00000001:1.0:1784949309.877708:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949309.877714:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949309.877716:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949309.877719:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1784949309.877720:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949309.877722:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:0.0:1784949309.877727:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000200:3.0:1784949309.877730:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949309.877733:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000040:1.0:1784949309.877737:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001e461ac8 x1871651289688448/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:215/0 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949309.877746:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949309.877754:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949309.877764:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949309.877770:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949309.877776:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949309.877785:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949309.877794:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000200:3.0:1784949309.877795:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949309.877799:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:1.0:1784949309.877802:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001e461ac8 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289688448:12345-0@lo:400: Request processed in 673us (1104us total) trans 0 rc 0/0 02000000:00000010:0.0:1784949309.877807:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000002f80d253. 02000000:00000001:0.0:1784949309.877817:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949309.877826:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949309.877835:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949309.877838:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 824 00010000:00000040:0.0:1784949309.877845:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289688576 00000100:00000040:1.0:1784949309.877849:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949309.877852:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00010000:00000001:0.0:1784949309.877854:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949309.877856:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949309.877858:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949309.877862:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00010000:00000200:0.0:1784949309.877866:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006e540bde x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:215/0 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949309.877867:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:1.0:1784949309.877872:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000030e14bca. 02000000:00000001:3.0:1784949309.877875:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949309.877880:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:1.0:1784949309.877884:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949309.877885:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949309.877897:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949309.877898:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000bbdbdf4. 00010000:00000001:0.0:1784949309.877899:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949309.877906:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949309.877911:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=31 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949309.877912:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=26 v=5 (1 1 1 1) 00000020:00000040:1.0:1784949309.877912:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:3.0:1784949309.877954:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=31 v=5 (1 1 1 1) 02000000:00000001:1.0:1784949309.877965:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949309.877971:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009e9a21e3. 00000100:00000001:3.0:1784949309.877972:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949309.877973:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949309.877977:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949309.877979:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949309.877984:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949309.877985:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949309.877988:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949309.877994:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949309.877996:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000200:1.0:1784949309.877997:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949309.878000:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949309.878009:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000400:00000010:1.0:1784949309.878012:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000036dd0276. 00000100:00000040:3.0:1784949309.878014:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949309.878015:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949309.878024:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784949309.878025:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000041929539. 00000100:00000040:0.0:1784949309.878038:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949309.878047:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784949309.878049:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a11d9749. 00000100:00000001:3.0:1784949309.878053:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000200:0.0:1784949309.878059:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289688576, offset 224 00000100:00000001:3.0:1784949309.878059:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949309.878071:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949309.878079:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:3.0:1784949309.878095:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289688448:0@lo:400: 00000400:00000200:0.0:1784949309.878103:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949309.878107:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949309.878112:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949309.878117:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3868:3868:242:4294967295] 192.168.202.137@tcp LPNI seq info [2476:2476:8:4294967295] 00000100:00000040:3.0:1784949309.878121:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000029a32fda x1871651289688448/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949309.878144:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:0.0:1784949309.878146:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:3.0:1784949309.878149:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949309.878155:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 00000800:00000200:0.0:1784949309.878161:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949309.878166:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000010:0.0:1784949309.878169:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003ed79484. 00000020:00000001:3.0:1784949309.878171:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949309.878176:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000800:00000200:0.0:1784949309.878184:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:3.0:1784949309.878185:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949309.878191:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000005678171. 02000000:00000001:3.0:1784949309.878197:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:0.0:1784949309.878199:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 02000000:00000001:3.0:1784949309.878201:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949309.878206:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000029a32fda. 00000800:00000200:0.0:1784949309.878209:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003ed79484 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949309.878213:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949309.878217:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949309.878236:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949309.878240:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949309.878242:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949309.878243:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949309.878244:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949309.878249:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949309.878253:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949309.878257:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949309.878259:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949309.878261:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949309.878267:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006e540bde x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:215/0 lens 224/224 e 0 to 0 dl 1784949320 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949309.878282:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006e540bde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289688576:12345-192.168.202.137@tcp:400: Request processed in 650us (1001us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949309.878292:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 800 00000100:00000040:0.0:1784949309.878295:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949309.878298:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949309.878299:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949309.878302:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000002fe80d84. 00000020:00000010:0.0:1784949309.878306:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000027aee7d. 00000020:00000010:0.0:1784949309.878309:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000208850ab. 00000020:00000040:0.0:1784949309.878313:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949309.878316:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949309.878350:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949309.878355:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003ed79484. 00000800:00000001:0.0:1784949309.878357:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949309.878360:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949309.878367:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949309.878368:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949309.878375:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949309.878377:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a11d9749 00000800:00000001:0.0:1784949309.878379:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949309.878403:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a11d9749. 00000400:00000200:0.0:1784949309.878412:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949309.878412:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949309.878417:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784949309.878426:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ea00 00000400:00000200:0.0:1784949309.878434:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x95cd [1] + 224 00000800:00000001:0.0:1784949309.878439:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949309.878448:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949309.878451:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949309.878455:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949309.878459:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949309.878462:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949309.878469:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949309.878483:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949309.878488:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949309.878492:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949309.878511:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949309.878516:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949309.878521:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949309.878523:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949309.878524:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bcdfbf21. 00000400:00000200:2.0:1784949309.878529:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:2.0:1784949309.878531:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000100:00000001:2.0:1784949309.878533:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949309.878538:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949309.878547:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949309.878553:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949309.878554:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949309.878556:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949309.878558:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949309.878559:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949309.878561:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949309.878563:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949309.878567:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=26 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949309.878570:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=26 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949309.878573:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949309.878574:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949309.878576:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949309.878579:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949309.878585:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949309.878588:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949309.878589:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949309.878591:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949309.878592:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949309.878595:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949309.878601:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949309.878603:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949309.878604:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949309.878606:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949309.878612:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289688576:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949309.878615:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949309.878617:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949309.878619:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005126bdfa x1871651289688576/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949309.878624:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949309.878626:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949309.878627:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000059e5b358. 02000000:00000001:2.0:1784949309.878630:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949309.878631:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949309.878632:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949309.878634:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949309.878635:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f495c80e. 02000000:00000001:2.0:1784949309.878637:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949309.878639:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949309.878640:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005126bdfa. 00000100:00000001:2.0:1784949309.878642:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949309.878643:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949309.878645:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949309.878648:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949309.878650:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949309.878652:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949309.878653:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949309.918955:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306249258, at 4306249259 00000100:00000010:0.0:1784949309.918962:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:1.0:1784949309.919967:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306249258, at 4306249260 00000100:00000010:1.0:1784949309.919974:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000040:2.0:1784949309.919980:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306249259, at 4306249260 00000100:00000040:3.0:1784949309.919981:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306249259, at 4306249260 00000100:00000010:2.0:1784949309.919990:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000010:3.0:1784949309.919992:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00010000:00000001:2.0:1784949310.708052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949310.708061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949310.708069:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949310.708071:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949310.708190:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949310.708197:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949310.708201:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949310.899997:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949310.900008:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949310.900009:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949310.900013:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949310.900016:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949310.900024:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949310.900028:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949311.732038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949311.732048:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949311.732056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949311.732060:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949311.732064:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949311.732067:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949311.732072:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949311.732074:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949311.732076:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949311.732079:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949311.732155:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949311.732161:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949311.732166:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949311.924120:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949311.924134:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949311.924137:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949311.924142:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949311.924145:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949311.924149:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949311.924152:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949312.756035:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949312.756044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949312.756071:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949312.756072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949312.756184:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949312.756189:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949312.756193:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949312.948032:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949312.948045:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949312.948048:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949312.948053:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949312.948057:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949312.948062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949312.948065:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949313.780001:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949313.780010:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949313.780020:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949313.780021:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949313.780091:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949313.780097:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949313.780102:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949313.972153:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949313.972172:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949313.972176:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949313.972182:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949313.972187:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949313.972191:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949313.972196:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949314.804048:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949314.804056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949314.804063:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949314.804065:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949314.804154:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949314.804160:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949314.804164:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1784949314.996029:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949314.996041:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949314.996046:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000010:3.0:1784949314.996050:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000100:00000001:0.0:1784949314.996051:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000040:3.0:1784949314.996059:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00100000:0.0:1784949314.996065:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949289/real 1784949289] req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949314 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949314.996067:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949314.996072:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949314.996080:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784949314.996085:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949314.996089:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000010:3.0:1784949314.996091:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000400:00000010:0.0:1784949314.996093:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000008b17e867. 00000400:00000200:0.0:1784949314.996101:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000100:00000001:3.0:1784949314.996102:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949314.996104:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000001:0.0:1784949314.996111:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1784949314.996113:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1784949314.996117:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949314 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949314.996141:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1784949314.996142:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949314 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949314.996146:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.996154:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949314.996156:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949314.996157:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_03+3] req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949314.996160:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949314.996163:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949314.996172:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949314 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949314.996204:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949314.996214:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949314.996215:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.996218:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949314.996221:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949314.996224:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949314.996228:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949314 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949314.996229:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949314.996236:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:1.0:1784949314.996238:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949314.996239:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949314.996246:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.996246:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1784949314.996246:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1784949314.996251:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949314.996253:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949314.996254:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949314.996254:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949314.996259:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1784949314.996263:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949314.996264:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1784949314.996264:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784949314.996271:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000010:3.0:1784949314.996271:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000020:00000040:3.0:1784949314.996279:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949314.996286:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949314.996289:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949314.996290:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1784949314.996292:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 02000000:00000001:3.0:1784949314.996294:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.996295:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.996298:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:3.0:1784949314.996303:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 02000000:00000001:1.0:1784949314.996305:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949314.996306:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949314 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949314.996311:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949314.996311:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949314.996320:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949314.996323:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1784949314.996324:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000009d6fa0dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289687424:192.168.202.137@tcp:8: 00000100:00100000:1.0:1784949314.996331:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289688704:0@lo:400: 00000100:00000001:0.0:1784949314.996336:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.996359:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949314.996360:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949314.996367:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000009d6fa0dc x1871651289687424/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949314 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949314.996367:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949314.996369:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949314.996372:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949314.996375:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949314.996385:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1784949314.996385:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_00_00+0] req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949314.996387:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949314.996392:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949314.996394:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000bbdbdf4. 02000000:00000010:0.0:1784949314.996396:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000077ad1613. 02000000:00000001:1.0:1784949314.996402:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949314.996404:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949314.996410:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949314.996412:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000400:00000010:1.0:1784949314.996414:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e17c5a5f. 00000020:00000001:0.0:1784949314.996415:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949314.996419:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000000b08bf97. 02000000:00000001:0.0:1784949314.996424:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949314.996425:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000010:1.0:1784949314.996425:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000100:00000001:3.0:1784949314.996427:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949314.996428:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000009d6fa0dc. 00000100:00000001:0.0:1784949314.996435:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000200:1.0:1784949314.996436:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289688704, portal 25 00000100:00000001:0.0:1784949314.996437:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949314.996437:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11592) 00000100:00000001:1.0:1784949314.996444:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949314.996445:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1784949314.996448:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:1.0:1784949314.996449:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000100:00000001:0.0:1784949314.996451:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949314.996454:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949314.996458:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949314.996459:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949314.996465:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949314.996465:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949314.996465:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1784949314.996470:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1784949314.996470:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784949314.996476:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949314.996480:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1784949314.996488:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11561 00000100:00000001:1.0:1784949314.996492:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949314.996499:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:3.0:1784949314.996502:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:0.0:1784949314.996505:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784949314.996507:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000001:0.0:1784949314.996508:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1784949314.996511:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11560 00000100:00000001:0.0:1784949314.996512:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000200:1.0:1784949314.996517:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289688704, offset 0 00000100:00000040:0.0:1784949314.996520:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949314.996529:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784949314.996530:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949314.996531:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949314.996536:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 02000000:00000001:0.0:1784949314.996538:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949314.996541:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949314.996544:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 02000000:00000001:0.0:1784949314.996545:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949314.996551:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289688832:192.168.202.137@tcp:400: 00000100:00000040:3.0:1784949314.996556:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000400:00000200:1.0:1784949314.996557:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949314.996559:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949314.996564:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949314.996565:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784949314.996566:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949314.996570:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1784949314.996572:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000010:0.0:1784949314.996575:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 00000400:00000200:1.0:1784949314.996579:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ea80 00000100:00000001:3.0:1784949314.996579:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:0.0:1784949314.996580:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949314.996588:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000005ecef72. 00000020:00000001:3.0:1784949314.996590:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949314.996593:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000200:1.0:1784949314.996597:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8c5d [2] + 0 00000020:00000001:3.0:1784949314.996599:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949314.996604:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:1.0:1784949314.996606:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000968c2fa4. 00000020:00000001:3.0:1784949314.996609:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:1.0:1784949314.996614:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000009412aeee 00000020:00000001:3.0:1784949314.996615:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:3.0:1784949314.996623:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000400:00000200:1.0:1784949314.996626:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949314.996627:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949314.996633:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000200:1.0:1784949314.996638:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949314.996639:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000020:00000040:3.0:1784949314.996645:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000200:1.0:1784949314.996648:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009412aeee 00000400:00000010:0.0:1784949314.996652:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009851728e. 00000100:00080000:3.0:1784949314.996653:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000010:1.0:1784949314.996654:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000009412aeee. 00000100:00080000:3.0:1784949314.996662:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000100:00000200:0.0:1784949314.996664:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289688832, portal 10 00000100:00000001:1.0:1784949314.996665:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:3.0:1784949314.996666:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949314.996671:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:0.0:1784949314.996672:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1784949314.996673:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1784949314.996676:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000100:00000040:1.0:1784949314.996680:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000322d6a4a x1871651289688704 msgsize 224 02000000:00000001:3.0:1784949314.996682:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949314.996687:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949314.996689:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949314.996696:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1784949314.996698:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784949314.996700:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949314.996706:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000010:3.0:1784949314.996710:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000100:00000001:0.0:1784949314.996712:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1784949314.996715:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:0.0:1784949314.996719:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949314.996722:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1784949314.996724:0:337122:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000001:3.0:1784949314.996725:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949314.996731:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949314.996733:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1784949314.996737:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 02000000:00000001:3.0:1784949314.996737:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949314.996743:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 02000000:00000010:3.0:1784949314.996748:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000cad84312. 00000100:00000001:1.0:1784949314.996750:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949314.996755:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949314.996756:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f59681f2. 00000100:00000200:1.0:1784949314.996758:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949314.996764:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289688832, offset 0 00000100:00100000:3.0:1784949314.996770:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949314.996783:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949314.996786:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949314.996788:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949314.996795:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949314.996796:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949314.996797:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949314.996802:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949314.996811:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:3.0:1784949314.996815:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_rcv+-1] req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949314.996817:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949314.996821:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1784949314.996823:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949314.996831:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:0.0:1784949314.996839:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1784949314.996842:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:0.0:1784949314.996846:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3869:3869:242:4294967295] 192.168.202.137@tcp LPNI seq info [2477:2477:8:4294967295] 00000100:00000001:3.0:1784949314.996849:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949314.996856:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11592) 00000400:00000200:0.0:1784949314.996858:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949314.996866:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949314.996870:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949314.996871:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000001:3.0:1784949314.996873:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949314.996880:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949314.996882:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306254378 00000100:00000001:3.0:1784949314.996884:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949314.996885:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000010:1.0:1784949314.996893:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000400:00000010:2.0:1784949314.996895:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000059e5b358. 00000100:00000040:3.0:1784949314.996896:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1784949314.996900:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008f5595bf. 00000400:00000010:2.0:1784949314.996925:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f071c401. 00000100:00000001:1.0:1784949314.996931:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000400:00000010:2.0:1784949314.996932:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000074c2f2ff. 00000100:00000001:1.0:1784949314.996942:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306254379 00000800:00000200:0.0:1784949314.996948:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784949314.996948:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000010:1.0:1784949314.996954:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:1.0:1784949314.996963:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949314.996963:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1784949314.996964:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000800:00000200:0.0:1784949314.996969:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:3.0:1784949314.996969:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.996973:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306254379 00000400:00000200:2.0:1784949314.996975:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:0.0:1784949314.996977:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f5595bf type 1, nob 320 niov 1 nkiov 1 02000000:00000001:3.0:1784949314.996978:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:1.0:1784949314.996981:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000007e14bd74. 00000400:00000200:2.0:1784949314.996984:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:3.0:1784949314.996984:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949314.996989:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000400:00000200:2.0:1784949314.996991:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949314.996992:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949314.996992:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.996997:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306254379 00000400:00000200:2.0:1784949314.996998:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000100:00000040:0.0:1784949314.997002:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2251 usecs (3 jiffies) 00000100:00000010:1.0:1784949314.997005:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000442a8405. 00000100:00100000:3.0:1784949314.997005:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289688960:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949314.997008:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306254340 00000400:00000200:2.0:1784949314.997008:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3870:3870:241:4294967295] 1.2.3.4@tcp LPNI seq info [117:117:8:4294967295] 00000100:00000001:1.0:1784949314.997015:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949314.997016:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306254378 00000100:00000001:3.0:1784949314.997018:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949314.997023:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306254340 00000100:00000001:1.0:1784949314.997024:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949314.997024:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000001:3.0:1784949314.997024:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.997032:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306254379 00000100:00000001:3.0:1784949314.997032:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.997033:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949314.997041:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:0.0:1784949314.997042:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306254340 00000100:00000001:1.0:1784949314.997042:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949314.997042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.997052:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306254379 00000100:00000001:3.0:1784949314.997052:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.997053:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949314.997059:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784949314.997061:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306254340 00000100:00000001:3.0:1784949314.997064:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949314.997065:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306254379 00000800:00000010:2.0:1784949314.997065:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000adf08f9b. 00000100:00000001:1.0:1784949314.997066:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949314.997069:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.997071:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949314.997074:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949314.997075:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1784949314.997078:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:1.0:1784949314.997079:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949314.997080:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949314.997080:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949314.997085:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949314.997087:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.997088:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.997089:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1784949314.997093:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:3.0:1784949314.997094:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.997095:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.997097:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949314.997103:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000001:0.0:1784949314.997104:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949314.997107:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:2.0:1784949314.997110:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100261 rc 0 00000100:00000001:0.0:1784949314.997112:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949314.997115:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949314.997118:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.997121:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.997130:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949314.997135:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289688704 02000000:00000001:1.0:1784949314.997138:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949314.997141:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949314.997144:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949314.997150:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949314.997154:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289688704 00000020:00000001:1.0:1784949314.997157:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949314.997159:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949314.997162:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949314.997165:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949314.997168:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784949314.997171:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949314.997176:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949314.997180:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000100:00000001:1.0:1784949314.997183:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949314.997187:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000020:00000010:1.0:1784949314.997195:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000020:00000010:1.0:1784949314.997201:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f505ae95. 00000020:00000010:1.0:1784949314.997206:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949314.997216:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:1.0:1784949314.997220:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949314.997223:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949314.997226:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949314.997227:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949314.997232:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008f5595bf. 00000100:00000001:1.0:1784949314.997235:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949314.997237:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949314.997259:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949314.997260:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949314.997265:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f59681f2 00000100:00000001:1.0:1784949314.997267:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949314.997267:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f59681f2. 00000100:00000001:3.0:1784949314.997271:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:0.0:1784949314.997272:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949314.997276:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784949314.997279:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949314.997282:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.997284:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949314.997289:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949314.997294:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1784949314.997296:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00100000:1.0:1784949314.997301:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 825 00000100:00000040:3.0:1784949314.997301:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949314.997309:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:0.0:1784949314.997312:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643eb00 00000100:00000001:1.0:1784949314.997316:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180541520 : -115539529010096 : ffff96eadb6a7450) 00000400:00000200:0.0:1784949314.997324:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 180464 00000100:00000001:3.0:1784949314.997324:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949314.997328:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000322d6a4a x1871651289688704/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:220/0 lens 224/0 e 0 to 0 dl 1784949325 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949314.997329:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784949314.997334:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949314.997357:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.997358:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949314.997362:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1784949314.997365:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000faccdd0d. 00000400:00000200:0.0:1784949314.997369:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949314.997371:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949314.997376:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000faccdd0d. 00000100:00000001:0.0:1784949314.997378:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:0.0:1784949314.997381:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00100000:1.0:1784949314.997385:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000322d6a4a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289688704:12345-0@lo:400: 00000100:00000010:0.0:1784949314.997388:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000009d6fa0dc. 00000100:00000200:1.0:1784949314.997392:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289688704 00000100:00000040:0.0:1784949314.997395:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000009d6fa0dc x1871651289688832 msgsize 224 00000020:00000001:1.0:1784949314.997398:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949314.997403:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00100000:0.0:1784949314.997408:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:1.0:1784949314.997409:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949314.997415:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949314.997418:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949314.997423:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949314.997426:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949314.997427:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:1.0:1784949314.997430:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949314.997434:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:0.0:1784949314.997438:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:1.0:1784949314.997438:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949314.997444:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:0.0:1784949314.997445:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.997451:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949314.997456:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949314.997456:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000010:1.0:1784949314.997461:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009e9a21e3. 00000100:00100000:2.0:1784949314.997462:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289688832 02000000:00000001:2.0:1784949314.997468:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:1.0:1784949314.997469:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949314.997471:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949314.997474:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949314.997476:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949314.997481:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949314.997481:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949314.997488:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289688832 00010000:00000040:1.0:1784949314.997489:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289688704 00000020:00000001:2.0:1784949314.997491:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949314.997493:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000001:1.0:1784949314.997496:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:2.0:1784949314.997497:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949314.997501:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000200:1.0:1784949314.997505:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000322d6a4a x1871651289688704/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:220/0 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949314.997505:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949314.997510:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949314.997516:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949314.997520:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:2.0:1784949314.997524:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1784949314.997525:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:2.0:1784949314.997526:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00010000:00000001:1.0:1784949314.997530:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949314.997533:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b1cbf66f. 00000100:00001000:1.0:1784949314.997536:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=36 v=5 (1 1 1 1) 00000020:00000010:2.0:1784949314.997537:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009d24ccac. 00000020:00000010:2.0:1784949314.997542:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784949314.997544:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949314.997549:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000040:2.0:1784949314.997550:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949314.997553:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:2.0:1784949314.997554:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949314.997556:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949314.997559:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949314.997560:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:2.0:1784949314.997564:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 02000000:00000001:1.0:1784949314.997565:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949314.997567:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949314.997571:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949314.997571:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949314.997578:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949314.997582:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000001:2.0:1784949314.997586:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949314.997587:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289688704, offset 224 00000400:00000200:1.0:1784949314.997597:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949314.997601:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949314.997605:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784949314.997609:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949314.997616:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:2.0:1784949314.997616:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 801 00000100:00000001:0.0:1784949314.997620:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784949314.997622:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ea80 00000100:00000001:0.0:1784949314.997624:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949314.997624:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949314.997631:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949314.997632:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x95e1 [1] + 224 00000100:00000001:2.0:1784949314.997632:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469560960 : -115544239990656 : ffff96e9c29e9880) 00000400:00000200:1.0:1784949314.997643:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949314.997643:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009d6fa0dc x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:220/0 lens 224/0 e 0 to 0 dl 1784949325 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949314.997653:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949314.997661:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949314.997667:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:1.0:1784949314.997668:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949314.997673:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000001:2.0:1784949314.997679:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949314.997685:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000040:1.0:1784949314.997691:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949314.997709:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009d6fa0dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289688832:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784949314.997719:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289688832 00000020:00000001:2.0:1784949314.997725:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949314.997729:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949314.997731:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949314.997733:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949314.997735:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949314.997737:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949314.997739:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949314.997741:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949314.997742:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949314.997743:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.997744:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1784949314.997745:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949314.997752:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949314.997754:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:1.0:1784949314.997755:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 02000000:00000010:2.0:1784949314.997757:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007e969be9. 00000400:00000010:1.0:1784949314.997760:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 02000000:00000001:2.0:1784949314.997762:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.997765:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949314.997766:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.997768:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949314.997768:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:2.0:1784949314.997770:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.997774:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00010000:00000040:2.0:1784949314.997775:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289688832 00000400:00000001:0.0:1784949314.997777:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00010000:00000001:2.0:1784949314.997779:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000001:0.0:1784949314.997780:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784949314.997783:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949314.997785:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e17c5a5f. 00010000:00000200:2.0:1784949314.997787:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009d6fa0dc x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:220/0 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949314.997792:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000100:00000040:1.0:1784949314.997794:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000010:0.0:1784949314.997797:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00010000:00000001:1.0:1784949314.997804:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:0.0:1784949314.997805:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:2.0:1784949314.997809:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000001:1.0:1784949314.997811:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949314.997813:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949314.997814:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949314.997817:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00001000:2.0:1784949314.997823:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=31 v=5 (1 1 1 1) 00000100:00000040:1.0:1784949314.997828:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000322d6a4a x1871651289688704/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:220/0 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949314.997851:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949314.997852:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949314.997864:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949314.997875:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949314.997885:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949314.997891:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949314.997902:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949314.997904:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949314.997909:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949314.997912:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000322d6a4a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289688704:12345-0@lo:400: Request processed in 547us (1224us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949314.997916:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000040:2.0:1784949314.997918:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949314.997927:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 02000000:00000001:0.0:1784949314.997961:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000200:2.0:1784949314.997966:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289688832, offset 224 00000100:00000001:0.0:1784949314.997968:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949314.997973:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949314.997980:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 825 02000000:00000001:0.0:1784949314.997983:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949314.997983:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949314.998007:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00001000:0.0:1784949314.998010:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=36 v=5 (5 5 5 5) 00000100:00000001:1.0:1784949314.998014:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000400:00000200:2.0:1784949314.998016:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00001000:0.0:1784949314.998018:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=36 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949314.998019:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:2.0:1784949314.998026:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3871:3871:241:4294967295] 192.168.202.137@tcp LPNI seq info [2478:2478:8:4294967295] 00000100:00000001:0.0:1784949314.998027:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000010:1.0:1784949314.998027:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f505ae95. 00000100:00000001:0.0:1784949314.998031:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949314.998036:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:1.0:1784949314.998037:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00010000:00000001:0.0:1784949314.998041:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949314.998045:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000100:00000001:0.0:1784949314.998047:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949314.998048:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:0.0:1784949314.998056:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949314.998056:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000800:00000200:2.0:1784949314.998074:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949314.998093:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949314.998094:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000800:00000010:2.0:1784949314.998094:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000081367938. 02000000:00000010:1.0:1784949314.998100:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000009e9a21e3. 00000100:00000001:0.0:1784949314.998101:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949314.998104:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949314.998106:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:1.0:1784949314.998108:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000040:0.0:1784949314.998110:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949314.998114:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949314.998116:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949314.998123:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000081367938 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:1.0:1784949314.998124:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:0.0:1784949314.998133:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289688704:0@lo:400: 00000400:00000010:1.0:1784949314.998135:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000772ddfc8. 00000100:00000001:0.0:1784949314.998140:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949314.998142:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.998143:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:1.0:1784949314.998144:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000471491fa. 00000100:00000040:0.0:1784949314.998151:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289688704/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949314.998151:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949314.998162:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949314.998166:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949314.998169:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1784949314.998170:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 02000000:00000001:0.0:1784949314.998174:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949314.998177:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000bbdbdf4. 00000100:00000040:2.0:1784949314.998179:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009d6fa0dc x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:220/0 lens 224/224 e 0 to 0 dl 1784949325 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949314.998181:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949314.998183:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949314.998186:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949314.998192:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949314.998195:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:0.0:1784949314.998200:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949314.998203:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949314.998206:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00100000:2.0:1784949314.998207:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009d6fa0dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289688832:12345-192.168.202.137@tcp:400: Request processed in 536us (808us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949314.998209:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949314.998212:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949314.998216:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949314.998223:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949314.998226:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949314.998227:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 801 00000100:00000001:0.0:1784949314.998229:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949314.998233:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949314.998235:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949314.998240:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949314.998243:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949314.998247:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009d24ccac. 00000020:00000010:2.0:1784949314.998253:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949314.998258:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b1cbf66f. 00000800:00000001:0.0:1784949314.998263:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000040:2.0:1784949314.998265:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000800:00000200:3.0:1784949314.998265:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949314.998271:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949314.998271:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949314.998272:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000081367938. 00000800:00000001:0.0:1784949314.998275:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949314.998281:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949314.998283:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949314.998289:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949314.998295:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949314.998301:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000200:0.0:1784949314.998303:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643eb00 00000400:00000010:3.0:1784949314.998305:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000100:00000001:3.0:1784949314.998311:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949314.998313:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:0.0:1784949314.998316:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x95e9 [1] + 224 00000800:00000001:0.0:1784949314.998322:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949314.998332:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949314.998334:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949314.998337:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949314.998357:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949314.998361:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949314.998367:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949314.998381:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949314.998385:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949314.998387:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.998429:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949314.998433:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949314.998438:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949314.998440:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949314.998442:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000005ecef72. 00000400:00000200:1.0:1784949314.998446:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000009851728e 00000400:00000010:1.0:1784949314.998448:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000009851728e. 00000100:00000001:1.0:1784949314.998451:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949314.998457:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949314.998468:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949314.998477:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949314.998479:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.998481:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949314.998483:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949314.998485:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949314.998487:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949314.998489:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949314.998494:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=31 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949314.998498:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=31 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949314.998500:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949314.998501:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949314.998504:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949314.998507:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949314.998516:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.998520:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949314.998521:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949314.998523:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949314.998525:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949314.998528:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949314.998536:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949314.998539:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949314.998540:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949314.998544:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949314.998552:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289688832:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949314.998558:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.998560:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949314.998563:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289688832/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949330 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949314.998571:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949314.998573:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949314.998576:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 02000000:00000001:1.0:1784949314.998582:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949314.998583:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949314.998585:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949314.998587:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949314.998589:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:1.0:1784949314.998592:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949314.998594:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949314.998595:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000100:00000001:1.0:1784949314.998607:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949314.998609:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.998611:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949314.998616:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949314.998617:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949314.998619:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949314.998621:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949315.039965:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306254379, at 4306254380 00000100:00000040:1.0:1784949315.039967:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306254379, at 4306254380 00000100:00000040:3.0:1784949315.039971:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306254379, at 4306254380 00000100:00000010:2.0:1784949315.039979:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000007e14bd74. 00000100:00000010:3.0:1784949315.039982:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000442a8405. 00000100:00000040:0.0:1784949315.039984:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306254378, at 4306254380 00000100:00000010:1.0:1784949315.039987:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:0.0:1784949315.039994:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00010000:00000001:2.0:1784949315.828010:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949315.828019:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949315.828028:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949315.828029:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949315.828112:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949315.828119:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949315.828123:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949316.020073:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949316.020082:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1784949316.020098:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949314/real 0] req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949315 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949316.020117:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949316.020121:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949316.020124:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949316.020129:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949315 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949316.020139:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949316.020150:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949316.020152:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949316.020156:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949316.020161:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949316.020164:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949316.020169:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949315 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949316.020190:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1784949316.020194:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949316.020197:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949316.020198:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949316.020201:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1784949316.020205:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1784949316.020210:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1784949316.020242:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:0.0:1784949316.020249:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1784949316.020256:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949315 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949316.020268:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289688960:1.2.3.4@tcp:8: 00000100:00000001:0.0:1784949316.020280:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949316.020285:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949316.020289:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289688960/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949315 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949316.020297:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949316.020300:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784949316.020307:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:2.0:1784949316.020310:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000020:00000040:0.0:1784949316.020313:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:0.0:1784949316.020319:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000010:2.0:1784949316.020320:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007316bde4. 02000000:00000010:0.0:1784949316.020325:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000cad84312. 00000020:00000040:2.0:1784949316.020329:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949316.020379:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949316.020383:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949316.020387:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949316.020391:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949316.020392:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949316.020397:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000100:00000001:0.0:1784949316.020404:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949316.020406:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1784949316.020407:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000031deb17e. 00000100:00000001:0.0:1784949316.020411:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.020419:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949316.020422:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949316.020429:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949316.020434:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949316.020442:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949316.020450:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.020459:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949316.020464:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949316.020472:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007316bde4] to pc [ptlrpcd_00_01+1] req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.020508:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949316.020513:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949316.020519:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784949316.020523:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1784949316.020527:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1784949316.020532:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1784949316.020537:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1784949316.020540:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11561 00000100:00000001:0.0:1784949316.020548:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949316.020553:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949316.020556:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949316.020560:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:1.0:1784949316.020568:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949316.020572:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1784949316.020574:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11587 00000400:00000200:2.0:1784949316.020602:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000001:0.0:1784949316.020604:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1784949316.020607:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1784949316.020609:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949316.020617:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:0.0:1784949316.020619:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949316.020624:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784949316.020626:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949316.020634:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:0.0:1784949316.020637:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.020643:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00100000:0.0:1784949316.020648:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007316bde4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289689088:0@lo:400: 00000020:00000001:2.0:1784949316.020652:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949316.020658:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:0.0:1784949316.020662:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1784949316.020664:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949316.020671:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:0.0:1784949316.020674:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1784949316.020676:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:0.0:1784949316.020681:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949316.020684:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:0.0:1784949316.020697:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784949316.020704:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000010:0.0:1784949316.020707:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004b465f54. 00000100:00000001:2.0:1784949316.020709:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1784949316.020717:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949316.020718:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949316.020728:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:2.0:1784949316.020732:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000010:0.0:1784949316.020733:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000055895529. 00000100:00080000:2.0:1784949316.020742:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000400:00000010:0.0:1784949316.020748:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e1a543c4. 00000100:00080000:2.0:1784949316.020749:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:2.0:1784949316.020759:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949316.020763:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1784949316.020764:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289689088, portal 25 00000100:00000001:0.0:1784949316.020770:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1784949316.020772:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949316.020773:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469580032 : -115544239971584 : ffff96e9c29ee300) 02000000:00000001:2.0:1784949316.020786:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.020791:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:0.0:1784949316.020792:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.020800:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949316.020813:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:2.0:1784949316.020817:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f37d62c3. 00000100:00000040:0.0:1784949316.020818:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949316.020827:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000020:00000040:2.0:1784949316.020828:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000200:0.0:1784949316.020834:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289689088, offset 0 02000000:00000001:2.0:1784949316.020840:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1784949316.020847:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:2.0:1784949316.020849:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.020858:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949316.020862:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949316.020867:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949316.020870:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:2.0:1784949316.020881:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000068fddc2b. 00000100:00000001:2.0:1784949316.020891:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949316.020893:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ec00 00000400:00000200:0.0:1784949316.020958:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8c79 [2] + 0 00000100:00100000:2.0:1784949316.020968:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1784949316.020971:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dee4e850. 00000400:00000200:0.0:1784949316.020989:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000097d07cda 00000100:00000001:2.0:1784949316.020994:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949316.021014:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949316.021020:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949316.021026:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f37d62c3] to pc [ptlrpcd_rcv+-1] req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949316.021040:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:0.0:1784949316.021059:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000097d07cda 00000400:00000010:0.0:1784949316.021069:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000097d07cda. 00000100:00000001:0.0:1784949316.021083:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784949316.021083:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949316.021091:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949316.021095:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1784949316.021102:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000030d4261 x1871651289689088 msgsize 224 00000100:00000001:2.0:1784949316.021120:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00100000:0.0:1784949316.021132:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000010:2.0:1784949316.021133:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003f023d19. 00000100:00000040:0.0:1784949316.021141:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000020:00000040:2.0:1784949316.021144:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949316.021153:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949316.021157:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949316.021161:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949316.021163:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949316.021166:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949316.021171:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1784949316.021181:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:1.0:1784949316.021184:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949316.021195:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949316.021196:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000100:00000040:1.0:1784949316.021198:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949316.021206:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:0.0:1784949316.021217:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1784949316.021220:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1784949316.021230:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949316.021248:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949316.021255:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949316.021266:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949316.021273:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949316.021285:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949316.021300:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f37d62c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289689216:192.168.202.137@tcp:8: 00000100:00000001:0.0:1784949316.021305:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949316.021305:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949316.021308:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949316.021316:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949316.021317:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1784949316.021319:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949316.021324:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:1.0:1784949316.021325:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949316.021365:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784949316.021371:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f023d19] to pc [ptlrpcd_00_02+2] req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949316.021374:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289689088 00000100:00000001:0.0:1784949316.021376:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949316.021382:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009e9a21e3. 02000000:00000001:3.0:1784949316.021384:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949316.021386:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949316.021394:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:1.0:1784949316.021395:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949316.021401:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.021402:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949316.021409:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000384f0ed7. 02000000:00000001:3.0:1784949316.021418:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949316.021419:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306255403 00000400:00000010:1.0:1784949316.021423:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000100:00000010:0.0:1784949316.021437:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000100:00000200:1.0:1784949316.021438:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289689216, portal 4 00000100:00000001:1.0:1784949316.021448:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949316.021453:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469571968 : -115544239979648 : ffff96e9c29ec380) 00000100:00000001:0.0:1784949316.021457:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949316.021464:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949316.021466:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949316.021473:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306255403 00000100:00000040:2.0:1784949316.021482:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11593) 00000100:00100000:3.0:1784949316.021484:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289689088 00000100:00000010:0.0:1784949316.021490:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000020:00000001:3.0:1784949316.021495:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949316.021501:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949316.021507:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949316.021510:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949316.021511:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949316.021511:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949316.021521:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306255403 00000020:00000040:3.0:1784949316.021522:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949316.021523:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949316.021529:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949316.021535:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784949316.021535:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:0.0:1784949316.021539:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 00000100:00000040:2.0:1784949316.021545:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949316.021546:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000010:1.0:1784949316.021548:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000100:00000001:0.0:1784949316.021554:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949316.021558:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:1.0:1784949316.021563:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289689216, offset 0 00000100:00000001:0.0:1784949316.021569:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306255403 00000100:00000001:2.0:1784949316.021583:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:0.0:1784949316.021589:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:2.0:1784949316.021589:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949316.021591:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:2.0:1784949316.021602:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949316.021608:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949316.021611:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 02000000:00000001:2.0:1784949316.021612:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949316.021616:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000486950f8. 00000400:00000200:1.0:1784949316.021620:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:3.0:1784949316.021628:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949316.021629:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784949316.021630:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003f023d19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289689344:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784949316.021638:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:3.0:1784949316.021639:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000486950f8. 00000100:00000001:0.0:1784949316.021646:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.021652:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949316.021656:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949316.021657:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:3.0:1784949316.021657:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f047fe3. 02000000:00000001:2.0:1784949316.021660:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949316.021664:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949316.021669:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949316.021674:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f029eaaf. 00000400:00000200:1.0:1784949316.021675:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1784949316.021676:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949316.021677:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949316.021685:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:1.0:1784949316.021690:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000010:2.0:1784949316.021700:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000b1cbf66f. 00000020:00000010:3.0:1784949316.021703:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949316.021706:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949316.021709:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949316.021714:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3872:3872:241:4294967295] 192.168.202.137@tcp LPNI seq info [2479:2479:8:4294967295] 00000100:00000001:0.0:1784949316.021715:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1784949316.021721:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bcdfbf21. 00000100:00000001:0.0:1784949316.021724:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949316.021725:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000400:00000010:2.0:1784949316.021734:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000100:00000001:3.0:1784949316.021736:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949316.021737:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.021743:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:2.0:1784949316.021749:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289689344, portal 10 00000100:00000001:3.0:1784949316.021751:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949316.021755:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:2.0:1784949316.021761:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949316.021765:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949316.021770:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469580928 : -115544239970688 : ffff96e9c29ee680) 00000100:00000001:3.0:1784949316.021771:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949316.021773:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949316.021784:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949316.021794:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949316.021795:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1784949316.021796:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000094ed938f. 00000800:00000200:1.0:1784949316.021820:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:3.0:1784949316.021822:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949316.021828:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949316.021831:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949316.021832:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949316.021834:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000200:1.0:1784949316.021842:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:3.0:1784949316.021851:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 826 00000800:00000200:1.0:1784949316.021853:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000094ed938f type 1, nob 616 niov 1 nkiov 1 00000100:00000040:2.0:1784949316.021854:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949316.021858:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949316.021861:0:342757:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.021867:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180535376 : -115539529016240 : ffff96eadb6a5c50) 00000100:00000001:0.0:1784949316.021869:0:342757:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949316.021869:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000100:00000001:1.0:1784949316.021882:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949316.021885:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289689344, offset 0 00000100:00000040:3.0:1784949316.021886:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000030d4261 x1871651289689088/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:222/0 lens 224/0 e 0 to 0 dl 1784949327 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949316.021892:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000400:00000200:2.0:1784949316.021944:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949316.021949:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306255403 00000400:00000200:2.0:1784949316.021961:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949316.021962:0:337118:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306255403 00000100:00000001:3.0:1784949316.021965:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949316.021975:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306255404 00000020:00000010:3.0:1784949316.021976:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000486950f8. 00000400:00000200:2.0:1784949316.021980:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784949316.021990:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949316.021993:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4306255403 00000400:00000200:2.0:1784949316.021995:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:1.0:1784949316.022004:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306255404 00000020:00000010:3.0:1784949316.022010:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000486950f8. 00000400:00000200:2.0:1784949316.022012:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1784949316.022021:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4306255403 00000400:00000200:2.0:1784949316.022023:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00100000:3.0:1784949316.022030:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000030d4261 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289689088:12345-0@lo:400: 00000100:00000001:1.0:1784949316.022035:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306255404 00000400:00000200:2.0:1784949316.022042:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3873:3873:240:4294967295] 192.168.202.137@tcp LPNI seq info [2480:2480:7:4294967295] 00000100:00000200:3.0:1784949316.022048:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289689088 00000100:00000001:1.0:1784949316.022049:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4306255403 00000020:00000001:3.0:1784949316.022060:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949316.022062:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949316.022070:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949316.022073:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949316.022077:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949316.022081:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949316.022087:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:2.0:1784949316.022089:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949316.022091:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949316.022094:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784949316.022105:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1784949316.022106:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000200:0.0:1784949316.022113:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:3.0:1784949316.022114:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949316.022115:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:2.0:1784949316.022116:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784949316.022127:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000010:0.0:1784949316.022129:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000094ed938f. 00000100:00000001:1.0:1784949316.022131:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1784949316.022132:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000010:2.0:1784949316.022133:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e79c6895. 00000100:00000001:1.0:1784949316.022140:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949316.022142:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949316.022143:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949316.022149:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949316.022154:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:3.0:1784949316.022155:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949316.022157:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.022163:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:0.0:1784949316.022168:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784949316.022172:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000200:2.0:1784949316.022178:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000200:0.0:1784949316.022179:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 02000000:00000010:3.0:1784949316.022182:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000ccf2f653. 00000800:00000200:2.0:1784949316.022190:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000e79c6895 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:0.0:1784949316.022191:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 02000000:00000001:3.0:1784949316.022192:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.022204:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949316.022207:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1784949316.022210:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949316.022216:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289689088 00000100:00000200:0.0:1784949316.022220:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.022222:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949316.022223:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:2.0:1784949316.022237:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2252 usecs (3 jiffies) 00010000:00000200:3.0:1784949316.022238:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000030d4261 x1871651289689088/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:222/0 lens 224/224 e 0 to 0 dl 1784949327 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.022248:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306255365 00000100:00000001:0.0:1784949316.022260:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949316.022262:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306255403 00000100:00000001:2.0:1784949316.022271:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306255365 00010000:00000001:3.0:1784949316.022272:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000040:0.0:1784949316.022276:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949341 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949316.022280:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.022283:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306255404 00000100:00000001:2.0:1784949316.022294:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306255365 00000100:00001000:3.0:1784949316.022294:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=38 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949316.022307:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306255404 00000100:00000001:3.0:1784949316.022313:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949316.022315:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949316.022316:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949316.022317:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306255365 00000100:00000040:3.0:1784949316.022321:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949316.022329:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949316.022330:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306255404 00000100:00000001:3.0:1784949316.022354:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949316.022360:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949316.022362:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.022366:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949316.022369:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949316.022377:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000800:00000001:1.0:1784949316.022379:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.022380:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000001:1.0:1784949316.022391:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949316.022392:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949316.022392:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289689088, offset 224 00000100:00000001:2.0:1784949316.022405:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.022411:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949316.022413:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:2.0:1784949316.022422:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949316.022429:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:3.0:1784949316.022436:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949316.022439:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949316.022444:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949316.022447:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.022453:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949316.022455:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ec00 00000400:00000200:3.0:1784949316.022467:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9601 [1] + 224 00000400:00000200:3.0:1784949316.022475:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949316.022481:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949316.022488:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949316.022495:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949316.022509:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949316.022539:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000200:0.0:1784949316.022547:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1784949316.022550:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000800:00000010:0.0:1784949316.022559:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e79c6895. 00000400:00000010:3.0:1784949316.022559:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000800:00000001:1.0:1784949316.022560:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949316.022573:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000400:00000200:0.0:1784949316.022578:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949316.022578:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949316.022581:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949316.022587:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949316.022591:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949316.022599:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784949316.022600:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949316.022603:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949316.022603:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.022608:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1784949316.022611:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:0.0:1784949316.022623:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000020:00000001:3.0:1784949316.022623:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949316.022633:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000020:00000001:3.0:1784949316.022634:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000200:1.0:1784949316.022635:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:3.0:1784949316.022646:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000030d4261 x1871651289689088/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:222/0 lens 224/224 e 0 to 0 dl 1784949327 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949316.022647:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:2.0:1784949316.022650:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949316.022657:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:0.0:1784949316.022663:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949316.022667:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000055895529. 00000400:00000200:1.0:1784949316.022669:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ed00 00000400:00000200:2.0:1784949316.022681:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e1a543c4 00000400:00000010:2.0:1784949316.022690:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e1a543c4. 00000100:00100000:3.0:1784949316.022708:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000030d4261 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289689088:12345-0@lo:400: Request processed in 728us (1590us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949316.022710:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949316.022711:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1784949316.022723:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 180688 00000100:00000040:0.0:1784949316.022727:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949316.022728:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949316.022737:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 826 00000800:00000001:1.0:1784949316.022743:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949316.022750:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949316.022760:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000001:1.0:1784949316.022764:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949316.022768:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:1.0:1784949316.022777:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1784949316.022779:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f029eaaf. 00000020:00000010:3.0:1784949316.022793:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000400:00000200:1.0:1784949316.022801:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949316.022803:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949316.022806:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f047fe3. 00000100:00000200:2.0:1784949316.022813:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949316.022816:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949316.022817:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000020:00000040:3.0:1784949316.022818:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:1.0:1784949316.022825:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000001:3.0:1784949316.022832:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000010:1.0:1784949316.022836:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006a448376. 02000000:00000010:3.0:1784949316.022839:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000ccf2f653. 00000100:00000040:1.0:1784949316.022848:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000006a448376 x1871651289689344 msgsize 224 00000800:00000001:0.0:1784949316.022853:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949316.022854:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949316.022862:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.022872:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:3.0:1784949316.022873:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00100000:1.0:1784949316.022879:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:0.0:1784949316.022883:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949316.022886:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949316.022887:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019827e71. 00000100:00000001:2.0:1784949316.022901:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:1.0:1784949316.022902:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:3.0:1784949316.022940:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002e1eb6f6. 00000800:00000001:1.0:1784949316.022954:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949316.022965:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643ec80 02000000:00000001:2.0:1784949316.022966:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000800:00000001:1.0:1784949316.022967:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.022976:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949316.022983:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.022992:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000100:0.0:1784949316.022993:0:337114:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289689216 offset 0 length 520: 4 02000000:00000001:2.0:1784949316.022995:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949316.022998:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289689344 02000000:00000001:3.0:1784949316.023008:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:0.0:1784949316.023011:0:337114:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 0000000023ed5eac is in inconsistent state, don't perform health checking (-2, 0) 00000100:00001000:2.0:1784949316.023012:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=38 v=5 (5 5 5 5) 00000100:00000001:3.0:1784949316.023013:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949316.023027:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949316.023031:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 02000000:00000001:3.0:1784949316.023038:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949316.023044:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=38 v=5 (1 1 1 1) 00000100:00100000:3.0:1784949316.023045:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289689344 00000020:00000001:3.0:1784949316.023051:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000001:0.0:1784949316.023052:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000040:3.0:1784949316.023059:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:2.0:1784949316.023060:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784949316.023066:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949316.023069:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949316.023072:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000800:00000001:0.0:1784949316.023073:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949316.023080:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949316.023084:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000800:00000001:0.0:1784949316.023085:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784949316.023088:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949316.023091:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000800:00000001:0.0:1784949316.023094:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.023099:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.023102:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949316.023110:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000486950f8. 00000100:00000001:3.0:1784949316.023116:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949316.023118:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949316.023122:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000486950f8. 00000020:00000010:3.0:1784949316.023132:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f047fe3. 00000020:00000010:3.0:1784949316.023140:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000f029eaaf. 00000020:00000010:3.0:1784949316.023148:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:2.0:1784949316.023156:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1784949316.023161:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949316.023165:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949316.023167:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949316.023175:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949316.023175:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949316.023180:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949316.023190:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000040:2.0:1784949316.023192:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949316.023194:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949316.023199:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949316.023220:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007316bde4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289689088:0@lo:400: 00000100:00000001:3.0:1784949316.023221:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949316.023238:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949316.023240:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949316.023243:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949316.023252:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1784949316.023255:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 802 00000100:00000040:3.0:1784949316.023262:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000040:2.0:1784949316.023268:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007316bde4 x1871651289689088/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949316.023269:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637802880 : -115544071748736 : ffff96e9cca5c380) 00000100:00000040:3.0:1784949316.023285:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006a448376 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:222/0 lens 224/0 e 0 to 0 dl 1784949327 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.023312:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1784949316.023318:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:2.0:1784949316.023320:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:3.0:1784949316.023324:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000486950f8. 02000000:00000010:2.0:1784949316.023326:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004b465f54. 00000100:00000001:3.0:1784949316.023355:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949316.023363:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000486950f8. 00000100:00100000:3.0:1784949316.023370:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006a448376 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289689344:12345-192.168.202.137@tcp:400: 02000000:00000001:2.0:1784949316.023374:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:3.0:1784949316.023376:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289689344 00000020:00000001:2.0:1784949316.023380:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:3.0:1784949316.023382:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000040:2.0:1784949316.023386:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949316.023389:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949316.023392:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:3.0:1784949316.023394:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949316.023397:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000031deb17e. 00000020:00000001:3.0:1784949316.023400:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:2.0:1784949316.023406:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1784949316.023407:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 02000000:00000001:2.0:1784949316.023414:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1784949316.023416:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000010:2.0:1784949316.023418:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007316bde4. 00000020:00000001:3.0:1784949316.023422:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.023425:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1784949316.023428:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949316.023430:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949316.023433:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:2.0:1784949316.023438:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949316.023439:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949316.023449:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949316.023454:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949316.023455:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:2.0:1784949316.023462:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949316.023463:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949316.023471:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1784949316.023472:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000ccf2f653. 00000100:00000001:2.0:1784949316.023475:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949316.023477:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.023483:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949316.023486:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949316.023488:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289689344 00010000:00000001:3.0:1784949316.023491:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949316.023497:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006a448376 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:222/0 lens 224/224 e 0 to 0 dl 1784949327 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949316.023507:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949316.023510:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949316.023512:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=33 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949316.023516:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949316.023519:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949316.023523:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949316.023526:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949316.023528:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949316.023530:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949316.023533:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949316.023536:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005f1dd6bd. 00000100:00000200:3.0:1784949316.023539:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289689344, offset 224 00000400:00000200:3.0:1784949316.023544:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949316.023553:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949316.023560:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3874:3874:241:4294967295] 192.168.202.137@tcp LPNI seq info [2481:2481:8:4294967295] 00000400:00000200:3.0:1784949316.023571:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949316.023577:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949316.023581:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949316.023587:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949316.023593:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949316.023597:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949316.023614:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949316.023623:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949316.023625:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949316.023628:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949316.023630:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949316.023633:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006a448376 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:222/0 lens 224/224 e 0 to 0 dl 1784949327 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949316.023646:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006a448376 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289689344:12345-192.168.202.137@tcp:400: Request processed in 325us (795us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949316.023655:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 802 00000100:00000040:3.0:1784949316.023659:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949316.023662:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949316.023665:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949316.023668:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000f029eaaf. 00000020:00000010:3.0:1784949316.023675:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949316.023678:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f047fe3. 00000020:00000040:3.0:1784949316.023682:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949316.023686:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949316.023737:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949316.023743:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:0.0:1784949316.023749:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949316.023756:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949316.023760:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005f1dd6bd 00000400:00000010:0.0:1784949316.023763:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000005f1dd6bd. 00000100:00000001:0.0:1784949316.023768:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949316.023771:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949316.023779:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949316.023787:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949316.023791:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949316.023795:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949316.023802:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949316.023814:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ed00 00000400:00000200:0.0:1784949316.023822:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9611 [1] + 224 00000800:00000001:0.0:1784949316.023828:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949316.023834:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949316.023836:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949316.023839:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949316.023844:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949316.023848:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949316.023860:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949316.023890:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949316.023898:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949316.023950:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.023952:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949316.023958:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949316.023964:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949316.023966:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949316.023969:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bcdfbf21. 00000400:00000200:2.0:1784949316.023975:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:2.0:1784949316.023979:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000100:00000001:2.0:1784949316.023983:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949316.023989:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949316.024000:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.024018:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949316.024019:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.024022:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949316.024024:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949316.024025:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949316.024027:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949316.024029:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949316.024035:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=33 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949316.024039:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=33 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949316.024042:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949316.024043:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949316.024046:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949316.024050:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949316.024058:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.024063:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949316.024065:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949316.024067:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949316.024069:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949316.024073:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.024080:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949316.024083:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949316.024084:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949316.024087:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949316.024095:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003f023d19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289689344:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949316.024100:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.024102:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949316.024104:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003f023d19 x1871651289689344/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949316.024112:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949316.024114:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949316.024116:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000b1cbf66f. 02000000:00000001:2.0:1784949316.024120:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949316.024121:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949316.024123:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949316.024125:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949316.024127:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 02000000:00000001:2.0:1784949316.024130:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949316.024132:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949316.024133:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003f023d19. 00000100:00000001:2.0:1784949316.024136:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949316.024138:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.024140:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949316.024145:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949316.024146:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949316.024149:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949316.024151:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949316.064049:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306255403, at 4306255404 00000100:00000010:0.0:1784949316.064063:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00000100:00000040:2.0:1784949316.065005:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306255404, at 4306255405 00000100:00000010:2.0:1784949316.065018:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:1.0:1784949316.065048:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306255404, at 4306255405 00000100:00000040:3.0:1784949316.065051:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306255404, at 4306255405 00000100:00000010:1.0:1784949316.065067:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00000100:00000010:3.0:1784949316.065071:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00010000:00000001:2.0:1784949316.852055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949316.852067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949316.852080:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949316.852083:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949316.852223:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949316.852230:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949316.852234:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949317.876016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949317.876022:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949317.876028:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949317.876030:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949317.876078:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949317.876083:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949317.876086:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949318.900044:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949318.900054:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949318.900062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949318.900064:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949318.900146:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949318.900153:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949318.900158:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1784949319.284042:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000800:00000010:3.0:1784949319.284051:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000200:3.0:1784949319.284063:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949319.284071:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949319.284080:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11592 health :900 00000400:00000200:3.0:1784949319.284087:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949319.284092:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:3.0:1784949319.284098:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:2.0:1784949319.284180:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949319.284194:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:2.0:1784949319.284205:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949319.284209:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949319.284214:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:2.0:1784949319.284219:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000400:00000200:2.0:1784949319.284227:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949319.284234:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:2.0:1784949319.284242:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002c1a3d41. 00000400:00000200:2.0:1784949319.284252:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:2.0:1784949319.284258:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:2.0:1784949319.284265:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00010000:00000001:2.0:1784949319.924037:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949319.924047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949319.924056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949319.924058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949319.924197:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949319.924206:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949319.924213:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:1.0:1784949320.308000:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949320.308011:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000400:00000010:1.0:1784949320.308022:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:1.0:1784949320.308028:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000400:00000010:1.0:1784949320.308040:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000da72cb2d. 00000400:00000200:1.0:1784949320.308045:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949320.308060:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949320.308068:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949320.308072:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3875:3875:242:4294967295] 192.168.202.137@tcp LPNI seq info [2482:2482:8:4294967295] 00000400:00000200:1.0:1784949320.308078:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949320.308084:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949320.308089:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949320.308093:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ca1124f5. 00000800:00000200:1.0:1784949320.308098:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949320.308105:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949320.308108:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ca1124f5 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949320.308124:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11594 health :900 00000800:00000001:1.0:1784949320.308258:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949320.308266:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784949320.308268:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784949320.308270:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ca1124f5. 00000800:00000001:1.0:1784949320.308272:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949320.308275:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949320.308278:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949320.308280:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784949320.308284:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949320.308285:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949320.308288:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:1.0:1784949320.308309:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784949320.308319:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784949320.308329:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949320.308333:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3876:3876:242:4294967295] 192.168.202.137@tcp LPNI seq info [2483:2483:8:4294967295] 00000400:00000200:1.0:1784949320.308342:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784949320.308346:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949320.308349:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000076916763. 00000800:00000200:1.0:1784949320.308353:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949320.308356:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949320.308358:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000076916763 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949320.308396:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784949320.308416:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949320.308419:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000076916763. 00000400:00000200:1.0:1784949320.308423:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949320.308427:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949320.308430:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949320.308432:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949320.308437:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949320.308439:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949320.308441:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949320.308447:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949320.308457:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9625 00000800:00000001:0.0:1784949320.308462:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949320.308465:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949320.308468:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949320.308470:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949320.308475:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:0.0:1784949320.308478:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000400:00000200:0.0:1784949320.308481:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949320.308484:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000010:0.0:1784949320.308487:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:0.0:1784949320.308491:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949320.308493:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949320.948003:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949320.948009:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949320.948014:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949320.948015:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949320.948099:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949320.948103:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949320.948106:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1784949321.139994:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949321.140009:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949321.140019:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003f023d19. 00000020:00000040:2.0:1784949321.140027:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949321.140033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949321.140035:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949321.140041:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784949321.140044:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949321.140048:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949321.140049:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.140055:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949321.140060:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1784949321.140063:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c2b2d4f6. 00000100:00000001:0.0:1784949321.140064:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.140069:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949321.140077:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949321.140092:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949321.140110:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949321.140114:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949321.140122:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f023d19] to pc [ptlrpcd_00_03+3] req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949321.140166:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949321.140171:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949321.140178:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949321.140183:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949321.140188:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949321.140191:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949321.140195:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1784949321.140195:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007316bde4. 00000100:00000001:0.0:1784949321.140200:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1784949321.140202:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000040:0.0:1784949321.140208:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949321.140211:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784949321.140215:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1784949321.140217:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949321.140222:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949321.140225:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.140232:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949321.140236:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949321.140237:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000031deb17e. 02000000:00000001:0.0:1784949321.140241:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949321.140246:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949321.140247:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949321.140251:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949321.140257:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003f023d19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289689472:0@lo:400: 00000100:00000040:2.0:1784949321.140262:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949321.140264:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1784949321.140273:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949321.140276:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949321.140280:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1784949321.140281:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949321.140287:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949321.140308:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949321.140313:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949321.140324:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c7491352. 00000100:00000040:2.0:1784949321.140327:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007316bde4] to pc [ptlrpcd_00_00+0] req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949321.140334:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00000200:0.0:1784949321.140346:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289689472, portal 25 00000100:00000001:0.0:1784949321.140350:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949321.140352:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469580928 : -115544239970688 : ffff96e9c29ee680) 00000100:00000040:0.0:1784949321.140359:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949321.140367:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949321.140370:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949321.140375:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000100:00000001:2.0:1784949321.140376:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949321.140379:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289689472, offset 0 00000100:00000040:2.0:1784949321.140386:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11598) 00000400:00000200:0.0:1784949321.140389:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949321.140392:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949321.140397:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949321.140399:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949321.140405:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949321.140406:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949321.140412:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949321.140413:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949321.140428:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949321.140430:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949321.140434:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784949321.140437:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ed80 02000000:00000001:3.0:1784949321.140438:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949321.140447:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949321.140453:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000007316bde4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289689600:192.168.202.137@tcp:400: 00000400:00000200:0.0:1784949321.140454:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8c99 [2] + 0 00000100:00000001:3.0:1784949321.140462:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:0.0:1784949321.140465:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009b268f62. 02000000:00000001:3.0:1784949321.140467:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949321.140471:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949321.140472:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000021ac459d 02000000:00000001:3.0:1784949321.140473:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949321.140495:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000009f047fe3. 02000000:00000001:3.0:1784949321.140498:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949321.140499:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:3.0:1784949321.140502:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fed0cd52. 00000400:00000200:0.0:1784949321.140506:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000010:3.0:1784949321.140508:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000400:00000200:0.0:1784949321.140511:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000021ac459d 00000400:00000010:0.0:1784949321.140514:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000021ac459d. 00000100:00000001:0.0:1784949321.140518:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949321.140519:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289689600, portal 10 00000100:00000200:0.0:1784949321.140520:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:3.0:1784949321.140522:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1784949321.140525:0:337122:0:(events.c:337:request_in_callback()) incoming req@00000000bcf1bd0a x1871651289689472 msgsize 224 00000100:00000001:3.0:1784949321.140525:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469580032 : -115544239971584 : ffff96e9c29ee300) 00000100:00100000:0.0:1784949321.140532:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784949321.140533:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949321.140535:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949321.140542:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949321.140544:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949321.140547:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000837f5ae5. 00000100:00000001:0.0:1784949321.140548:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:3.0:1784949321.140550:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289689600, offset 0 00000400:00000200:3.0:1784949321.140554:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949321.140557:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:0.0:1784949321.140560:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00000100:00000001:0.0:1784949321.140564:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1784949321.140565:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000200:0.0:1784949321.140571:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949321.140572:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949321.140579:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949321.140583:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784949321.140584:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949321.140589:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:0.0:1784949321.140590:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949321.140596:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3877:3877:242:4294967295] 192.168.202.137@tcp LPNI seq info [2484:2484:8:4294967295] 00000100:00000001:1.0:1784949321.140600:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949321.140605:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289689472 00000100:00000001:0.0:1784949321.140607:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949321.140608:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949321.140612:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949321.140613:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949321.140617:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949321.140617:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 02000000:00000001:1.0:1784949321.140622:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.140626:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949321.140626:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00100000:1.0:1784949321.140628:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289689472 00000020:00000001:1.0:1784949321.140634:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000010:3.0:1784949321.140635:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000100:00000001:0.0:1784949321.140636:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306260522 00000020:00000040:1.0:1784949321.140636:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949321.140641:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:0.0:1784949321.140644:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000086461120. 00000800:00000200:3.0:1784949321.140645:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000040:1.0:1784949321.140646:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949321.140652:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949321.140655:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1784949321.140655:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000200:3.0:1784949321.140661:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:1.0:1784949321.140664:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949321.140665:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306260522 00000800:00000200:3.0:1784949321.140666:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000020:00000010:1.0:1784949321.140668:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007d5987e9. 00000100:00000010:0.0:1784949321.140673:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000100:00000001:1.0:1784949321.140674:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949321.140679:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:1.0:1784949321.140679:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007d5987e9. 00000100:00000001:3.0:1784949321.140683:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.140684:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306260522 00000020:00000010:1.0:1784949321.140685:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000229221cc. 00000100:00000040:3.0:1784949321.140688:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2253 usecs (3 jiffies) 00000100:00000001:3.0:1784949321.140692:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306260483 00000100:00000010:0.0:1784949321.140693:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000421f7b34. 00000020:00000010:1.0:1784949321.140693:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000b1b47c1. 00000100:00000001:3.0:1784949321.140696:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306260522 00000100:00000001:0.0:1784949321.140700:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:1.0:1784949321.140701:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949321.140701:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306260483 00000100:00000001:3.0:1784949321.140707:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306260522 00000100:00000001:0.0:1784949321.140709:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306260522 00000100:00000001:3.0:1784949321.140710:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306260483 00000100:00000001:3.0:1784949321.140714:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306260522 00000100:00000010:0.0:1784949321.140715:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000040:1.0:1784949321.140715:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949321.140716:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306260483 00000100:00000001:1.0:1784949321.140720:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949321.140723:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949321.140724:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949321.140727:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.140730:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949321.140731:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306260522 00000100:00000001:1.0:1784949321.140735:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949321.140736:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949321.140737:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949321.140743:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.140744:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949321.140746:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949321.140748:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949321.140748:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949321.140754:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949321.140757:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949321.140757:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949321.140758:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949321.140761:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949321.140762:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949321.140762:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949321.140767:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949321.140768:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949321.140770:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949321.140771:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1784949321.140773:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 827 00000100:00000001:0.0:1784949321.140775:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949321.140775:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949321.140777:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949321.140780:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949321.140781:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180531280 : -115539529020336 : ffff96eadb6a4c50) 00000100:00000001:0.0:1784949321.140782:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949321.140790:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bcf1bd0a x1871651289689472/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:227/0 lens 224/0 e 0 to 0 dl 1784949332 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949321.140798:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949321.140801:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000007d5987e9. 00000100:00000001:1.0:1784949321.140803:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949321.140805:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000007d5987e9. 00000100:00100000:1.0:1784949321.140809:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bcf1bd0a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289689472:12345-0@lo:400: 00000100:00000200:1.0:1784949321.140812:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289689472 00000020:00000001:1.0:1784949321.140814:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949321.140818:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949321.140819:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949321.140821:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949321.140823:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949321.140826:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949321.140828:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949321.140831:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000200:2.0:1784949321.140833:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:1.0:1784949321.140834:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:3.0:1784949321.140836:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:2.0:1784949321.140838:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000020:00000001:1.0:1784949321.140839:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949321.140843:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:2.0:1784949321.140846:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949321.140847:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000001:3.0:1784949321.140847:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949321.140852:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:3.0:1784949321.140853:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949321.140857:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000010:1.0:1784949321.140858:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000049c99d00. 00000800:00000001:3.0:1784949321.140859:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949321.140863:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949321.140863:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000837f5ae5 00000400:00000010:2.0:1784949321.140868:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000837f5ae5. 00000100:00000001:1.0:1784949321.140869:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949321.140870:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:1.0:1784949321.140890:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949321.140891:0:337114:0:(events.c:34:request_out_callback()) Process entered 00010000:00000040:1.0:1784949321.140896:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289689472 00000100:00000200:2.0:1784949321.140904:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949321.140904:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ee00 00010000:00000001:1.0:1784949321.140905:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949321.140914:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bcf1bd0a x1871651289689472/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:227/0 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949321.140947:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784949321.140952:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 180912 00000100:00000040:2.0:1784949321.140953:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949321.140989:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949321.140997:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949321.141002:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1784949321.141007:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949321.141008:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949321.141011:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949321.141012:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784949321.141012:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00001000:1.0:1784949321.141015:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=43 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949321.141020:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949321.141024:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949321.141024:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949321.141025:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000200:3.0:1784949321.141027:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 02000000:00000001:1.0:1784949321.141029:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949321.141032:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949321.141033:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d868603. 00000100:00000001:1.0:1784949321.141035:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949321.141037:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000006d868603 x1871651289689600 msgsize 224 00000100:00000040:1.0:1784949321.141041:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949321.141047:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000100:00100000:3.0:1784949321.141048:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:1.0:1784949321.141053:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289689472, offset 224 00000400:00000200:1.0:1784949321.141060:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949321.141062:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784949321.141069:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000001:3.0:1784949321.141070:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949321.141075:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949321.141079:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ed80 00000100:00000001:2.0:1784949321.141084:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784949321.141085:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9629 [1] + 224 00000100:00100000:2.0:1784949321.141088:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289689600 00000400:00000200:1.0:1784949321.141091:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1784949321.141092:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949321.141095:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949321.141098:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949321.141099:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:2.0:1784949321.141103:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949321.141105:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949321.141107:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289689600 00000100:00000200:1.0:1784949321.141110:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949321.141110:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949321.141113:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949321.141117:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949321.141121:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000040:1.0:1784949321.141123:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949321.141126:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949321.141131:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949321.141137:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949321.141141:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:1.0:1784949321.141143:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949321.141144:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949321.141147:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000400:00000200:1.0:1784949321.141150:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000400:00000010:1.0:1784949321.141152:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000020:00000010:2.0:1784949321.141152:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b1cbf66f. 00000100:00000001:0.0:1784949321.141156:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1784949321.141159:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009d24ccac. 00000100:00000001:0.0:1784949321.141160:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949321.141160:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949321.141164:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000010:2.0:1784949321.141165:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000001:0.0:1784949321.141167:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784949321.141169:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784949321.141170:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949321.141173:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c7491352. 00000100:00000040:1.0:1784949321.141174:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000040:2.0:1784949321.141175:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00010000:00000001:1.0:1784949321.141179:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:0.0:1784949321.141181:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000100:00000001:2.0:1784949321.141181:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000001:1.0:1784949321.141182:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949321.141186:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:2.0:1784949321.141186:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000020:00000001:1.0:1784949321.141187:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949321.141192:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949321.141192:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949321.141194:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bcf1bd0a x1871651289689472/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:227/0 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949321.141199:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949321.141201:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949321.141222:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949321.141227:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949321.141244:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bcf1bd0a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289689472:12345-0@lo:400: Request processed in 441us (713us total) trans 0 rc 0/0 00000100:00000200:0.0:1784949321.141249:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949321.141250:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949321.141260:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 827 00000100:00000040:1.0:1784949321.141285:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949321.141287:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949321.141287:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949321.141302:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949321.141302:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949321.141303:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949321.141307:0:337122:0:(client.c:1580:after_reply()) Process entered 00000100:00000001:1.0:1784949321.141308:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949321.141309:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000001:0.0:1784949321.141310:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:1.0:1784949321.141313:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000b1b47c1. 00000100:00000001:0.0:1784949321.141314:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949321.141314:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949321.141315:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 803 00000100:00000001:0.0:1784949321.141318:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949321.141321:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 02000000:00000001:0.0:1784949321.141322:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949321.141323:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949321.141323:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949321.141326:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000229221cc. 00000100:00000001:3.0:1784949321.141329:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949321.141330:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=43 v=5 (5 5 5 5) 00000100:00000001:2.0:1784949321.141330:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554951680 : -115544154599936 : ffff96e9c7b58e00) 00000020:00000040:1.0:1784949321.141332:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00001000:0.0:1784949321.141339:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=43 v=5 (1 1 1 1) 00000100:00000040:2.0:1784949321.141339:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006d868603 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:227/0 lens 224/0 e 0 to 0 dl 1784949332 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949321.141340:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949321.141344:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000049c99d00. 00000100:00000001:0.0:1784949321.141347:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000001:1.0:1784949321.141349:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949321.141350:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949321.141353:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784949321.141356:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784949321.141360:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949321.141360:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784949321.141362:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000010:1.0:1784949321.141366:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e17c5a5f. 00000100:00000001:0.0:1784949321.141367:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949321.141367:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000008648a444. 00000100:00000001:2.0:1784949321.141373:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784949321.141389:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b9b3f73e. 00000100:00000040:0.0:1784949321.141393:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949321.141393:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000008648a444. 00000100:00000001:0.0:1784949321.141407:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784949321.141407:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006d868603 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289689600:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949321.141415:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000200:2.0:1784949321.141416:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289689600 00000100:00000001:0.0:1784949321.141419:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949321.141421:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000040:0.0:1784949321.141425:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949321.141426:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949321.141430:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949321.141436:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949321.141455:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00100000:0.0:1784949321.141458:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003f023d19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289689472:0@lo:400: 00000020:00000001:2.0:1784949321.141459:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949321.141464:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949321.141464:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949321.141469:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949321.141469:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:0.0:1784949321.141472:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003f023d19 x1871651289689472/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949321.141472:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949321.141475:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949321.141479:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949321.141482:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949321.141483:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949321.141485:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:0.0:1784949321.141486:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949321.141489:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000010:2.0:1784949321.141489:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000cc641d8a. 02000000:00000001:0.0:1784949321.141492:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1784949321.141492:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949321.141494:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949321.141496:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:2.0:1784949321.141497:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949321.141498:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00010000:00000040:2.0:1784949321.141500:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289689600 00000020:00000001:0.0:1784949321.141501:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00010000:00000001:2.0:1784949321.141503:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000010:0.0:1784949321.141505:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c2b2d4f6. 00010000:00000200:2.0:1784949321.141508:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006d868603 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:227/0 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949321.141510:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949321.141512:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949321.141515:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003f023d19. 00000100:00000001:0.0:1784949321.141519:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00010000:00000001:2.0:1784949321.141520:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949321.141522:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949321.141523:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949321.141525:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:2.0:1784949321.141526:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=38 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949321.141529:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949321.141530:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949321.141532:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949321.141534:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949321.141536:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949321.141538:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:0.0:1784949321.141539:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949321.141541:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949321.141543:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949321.141545:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949321.141546:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949321.141549:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000100:00000200:2.0:1784949321.141552:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289689600, offset 224 00000400:00000200:2.0:1784949321.141556:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949321.141561:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949321.141564:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3878:3878:242:4294967295] 192.168.202.137@tcp LPNI seq info [2485:2485:8:4294967295] 00000400:00000200:2.0:1784949321.141570:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949321.141573:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949321.141575:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ffbbc320. 00000800:00000200:2.0:1784949321.141578:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949321.141581:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949321.141583:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ffbbc320 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949321.141593:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949321.141594:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949321.141596:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949321.141597:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949321.141598:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949321.141600:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006d868603 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:227/0 lens 224/224 e 0 to 0 dl 1784949332 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949321.141606:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006d868603 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289689600:12345-192.168.202.137@tcp:400: Request processed in 244us (568us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949321.141613:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 803 00000100:00000040:2.0:1784949321.141615:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949321.141617:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949321.141618:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949321.141620:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009d24ccac. 00000020:00000010:2.0:1784949321.141622:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949321.141625:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b1cbf66f. 00000020:00000040:2.0:1784949321.141628:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949321.141631:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949321.141675:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949321.141679:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ffbbc320. 00000800:00000001:2.0:1784949321.141681:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949321.141682:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949321.141688:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949321.141691:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000010:0.0:1784949321.141692:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000800:00000001:2.0:1784949321.141692:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949321.141697:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:2.0:1784949321.141698:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949321.141699:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949321.141704:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949321.141709:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949321.141716:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ee00 00000400:00000200:2.0:1784949321.141719:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9631 [1] + 224 00000800:00000001:2.0:1784949321.141722:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949321.141728:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949321.141729:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949321.141732:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949321.141735:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949321.141737:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949321.141743:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949321.141754:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949321.141758:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949321.141760:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949321.141776:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949321.141778:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949321.141780:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949321.141781:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949321.141783:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000fed0cd52. 00000400:00000200:3.0:1784949321.141787:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949321.141788:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:3.0:1784949321.141790:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949321.141793:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949321.141800:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949321.141804:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949321.141805:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949321.141806:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949321.141807:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949321.141808:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949321.141809:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949321.141810:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949321.141814:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=38 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949321.141816:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=38 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949321.141817:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949321.141819:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949321.141820:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949321.141822:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949321.141827:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949321.141829:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949321.141830:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949321.141831:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949321.141832:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949321.141834:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949321.141838:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949321.141839:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949321.141840:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949321.141842:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949321.141846:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000007316bde4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289689600:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949321.141848:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949321.141849:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949321.141850:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000007316bde4 x1871651289689600/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949321.141854:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949321.141855:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949321.141856:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000009f047fe3. 02000000:00000001:3.0:1784949321.141858:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949321.141859:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949321.141859:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949321.141861:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949321.141862:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000031deb17e. 02000000:00000001:3.0:1784949321.141864:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949321.141864:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949321.141865:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007316bde4. 00000100:00000001:3.0:1784949321.141867:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949321.141868:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949321.141869:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949321.141871:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949321.141872:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949321.141873:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949321.141874:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949321.182963:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306260522, at 4306260523 00000100:00000040:3.0:1784949321.182975:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306260522, at 4306260523 00000100:00000010:2.0:1784949321.182978:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000421f7b34. 00000100:00000010:3.0:1784949321.182984:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000040:1.0:1784949321.182987:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306260522, at 4306260523 00000100:00000040:0.0:1784949321.182994:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306260522, at 4306260523 00000100:00000010:1.0:1784949321.182996:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:0.0:1784949321.183005:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000086461120. 00010000:00000001:2.0:1784949321.971972:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949321.971979:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949321.971984:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949321.971986:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949321.971989:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949321.971991:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949321.971994:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949321.971996:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949321.971997:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949321.971998:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949321.972087:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949321.972091:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949321.972094:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949322.164129:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949322.164146:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949322.164176:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949322.164183:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949322.164186:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949322.164190:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949322.164193:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949322.356072:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000800:00000010:3.0:1784949322.356081:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fee8262a. 00000400:00000200:3.0:1784949322.356093:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949322.356104:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949322.356113:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11595 health :900 00000400:00000200:3.0:1784949322.356119:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949322.356124:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:3.0:1784949322.356129:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:2.0:1784949322.356194:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949322.356206:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:2.0:1784949322.356215:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949322.356218:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949322.356222:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:2.0:1784949322.356225:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000400:00000200:2.0:1784949322.356232:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949322.356237:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:2.0:1784949322.356242:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005152bcfe. 00000400:00000200:2.0:1784949322.356248:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:2.0:1784949322.356252:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:2.0:1784949322.356256:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00010000:00000001:2.0:1784949322.996052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949322.996062:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949322.996070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949322.996072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949322.996220:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949322.996226:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949322.996231:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949323.188042:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949323.188058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949323.188061:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949323.188067:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949323.188070:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949323.188074:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949323.188078:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949323.380030:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000800:00000010:3.0:1784949323.380039:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e4c24e27. 00000400:00000200:3.0:1784949323.380053:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949323.380063:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949323.380077:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949323.380084:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:3.0:1784949323.380086:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:3.0:1784949323.380097:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:1.0:1784949323.380101:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000400:00000010:1.0:1784949323.380138:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000229221cc. 00000400:00000010:1.0:1784949323.380147:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000800:00000100:3.0:1784949323.380149:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:3.0:1784949323.380157:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000af953512. 00000400:00000010:1.0:1784949323.380164:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001fcbecf4. 00000400:00000200:3.0:1784949323.380165:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949323.380169:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949323.380186:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000200:1.0:1784949323.380189:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949323.380190:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949323.380201:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11596 health :700 00000400:00000001:2.0:1784949323.380215:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949323.380222:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949323.380227:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000400:00000010:2.0:1784949323.380233:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000400:00000200:2.0:1784949323.380242:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949323.380243:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949323.380257:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:3.0:1784949323.380257:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000010:2.0:1784949323.380296:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000400:00000200:2.0:1784949323.380311:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:2.0:1784949323.380317:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:1.0:1784949323.380588:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949323.380598:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949323.380604:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3879:3879:244:4294967295] 192.168.202.137@tcp LPNI seq info [2486:2486:8:4294967295] 00000400:00000200:1.0:1784949323.380614:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949323.380623:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949323.380630:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949323.380636:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000800:00000010:1.0:1784949323.380642:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076916763. 00000400:00000200:2.0:1784949323.380653:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000800:00000200:1.0:1784949323.380661:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784949323.380680:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000800:00000200:1.0:1784949323.380682:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:2.0:1784949323.380692:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1784949323.380693:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000076916763 type 2, nob 96 niov 1 nkiov 0 00000400:00000001:2.0:1784949323.380699:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949323.380705:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000010:2.0:1784949323.380709:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000400:00000200:2.0:1784949323.380717:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949323.380728:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:2.0:1784949323.380735:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000047e19e91. 00000400:00000200:2.0:1784949323.380742:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000400:00000200:2.0:1784949323.380748:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000400:00000200:2.0:1784949323.380752:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000800:00000200:3.0:1784949323.380856:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949323.380862:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000076916763. 00000400:00000200:3.0:1784949323.380868:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949323.380875:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949323.380880:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949323.380884:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949323.380896:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949323.380944:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949323.380949:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949323.380957:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949323.380964:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949323.380980:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949323.380991:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949323.381003:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949323.381009:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3880:3880:245:4294967295] 192.168.202.137@tcp LPNI seq info [2487:2487:8:4294967295] 00000400:00000200:3.0:1784949323.381020:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949323.381026:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949323.381030:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000800:00000200:3.0:1784949323.381036:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949323.381043:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949323.381047:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 3, nob 144 niov 1 nkiov 1 00000800:00000001:2.0:1784949323.381114:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949323.381162:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949323.381168:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000400:00000200:3.0:1784949323.381174:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949323.381181:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949323.381184:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949323.381204:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949323.381210:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949323.381211:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949323.381222:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949323.381242:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949323.381266:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9645 00000800:00000001:0.0:1784949323.381302:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949323.381322:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949323.381328:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949323.381333:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949323.381339:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000400:00000010:0.0:1784949323.381343:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000400:00000200:0.0:1784949323.381346:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949323.381350:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000010:0.0:1784949323.381354:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000229221cc. 00000800:00000001:0.0:1784949323.381362:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949323.381365:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949324.020058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949324.020087:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949324.020098:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949324.020100:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949324.020339:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949324.020344:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949324.020347:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949324.212033:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949324.212050:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949324.212052:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949324.212058:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949324.212062:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949324.212067:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949324.212070:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949324.404133:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000800:00000010:3.0:1784949324.404141:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a1bdc144. 00000400:00000200:3.0:1784949324.404153:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949324.404160:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949324.404169:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11597 health :900 00000400:00000200:3.0:1784949324.404176:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949324.404181:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000200:3.0:1784949324.404187:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:2.0:1784949324.404320:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949324.404329:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x861 00000400:00000001:2.0:1784949324.404336:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949324.404339:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949324.404343:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:2.0:1784949324.404345:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000400:00000200:2.0:1784949324.404350:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949324.404354:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:2.0:1784949324.404358:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef8e57c8. 00000400:00000200:2.0:1784949324.404365:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:2.0:1784949324.404368:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:2.0:1784949324.404371:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00010000:00000001:2.0:1784949325.044050:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949325.044059:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949325.044068:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949325.044070:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949325.044201:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949325.044208:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949325.044212:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949325.236066:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949325.236076:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949325.236078:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949325.236083:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949325.236086:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949325.236089:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949325.236091:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949325.428001:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949325.428011:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000400:00000010:1.0:1784949325.428022:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000bbdbdf4. 00000400:00000010:1.0:1784949325.428028:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000400:00000010:1.0:1784949325.428035:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000056670828. 00000400:00000200:1.0:1784949325.428038:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949325.428052:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949325.428058:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949325.428061:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3881:3881:246:4294967295] 192.168.202.137@tcp LPNI seq info [2488:2488:8:4294967295] 00000400:00000200:1.0:1784949325.428067:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949325.428081:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949325.428085:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949325.428089:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:1.0:1784949325.428096:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949325.428104:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949325.428108:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009e7fe55b type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949325.428124:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11599 health :900 00000800:00000200:3.0:1784949325.428240:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949325.428246:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:3.0:1784949325.428303:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949325.428309:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949325.428314:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949325.428317:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949325.428327:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949325.428358:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949325.428361:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949325.428365:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949325.428372:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949325.428383:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949325.428391:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949325.428398:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949325.428402:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3882:3882:246:4294967295] 192.168.202.137@tcp LPNI seq info [2489:2489:8:4294967295] 00000400:00000200:2.0:1784949325.428408:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949325.428412:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949325.428415:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000004c284093. 00000800:00000200:2.0:1784949325.428419:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949325.428422:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949325.428424:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000004c284093 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949325.428515:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949325.428516:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949325.428520:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000004c284093. 00000800:00000001:0.0:1784949325.428525:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949325.428525:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949325.428530:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949325.428534:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949325.428538:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949325.428553:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949325.428557:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949325.428563:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949325.428572:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9649 00000800:00000001:3.0:1784949325.428576:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949325.428581:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949325.428582:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949325.428585:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949325.428589:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:3.0:1784949325.428591:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000400:00000200:3.0:1784949325.428593:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949325.428594:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000010:3.0:1784949325.428597:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000bbdbdf4. 00000800:00000001:3.0:1784949325.428602:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949325.428603:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949326.068016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949326.068022:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949326.068028:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949326.068029:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.068102:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949326.068106:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949326.068109:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949326.260074:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949326.260092:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949326.260095:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949326.260100:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949326.260104:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949326.260110:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949326.260114:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949326.260151:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949326.260161:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949326.260171:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000020:00000040:3.0:1784949326.260181:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949326.260188:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949326.260193:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949326.260201:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949326.260212:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003547cd5f. 00000100:00000001:3.0:1784949326.260222:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949326.260235:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949326.260297:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949326.260302:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949326.260315:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbc2d61e] to pc [ptlrpcd_00_01+1] req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949326.260368:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949326.260370:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.260378:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1784949326.260381:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:2.0:1784949326.260384:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1784949326.260394:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949326.260399:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949326.260402:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949326.260417:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949326.260426:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000100:00000001:2.0:1784949326.260428:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1784949326.260435:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949326.260436:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949326.260440:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949326.260446:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949326.260450:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949326.260451:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949326.260451:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949326.260459:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949326.260461:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949326.260471:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949326.260476:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000a929270c. 00000100:00100000:2.0:1784949326.260478:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289689728:0@lo:400: 00000100:00000001:3.0:1784949326.260487:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.260490:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949326.260500:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784949326.260504:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1784949326.260517:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949326.260527:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949326.260536:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949326.260538:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949326.260539:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000b1cbf66f. 00000100:00000040:3.0:1784949326.260545:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_00_02+2] req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949326.260548:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949326.260564:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bcdfbf21. 00000400:00000010:2.0:1784949326.260582:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000100:00000200:2.0:1784949326.260594:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289689728, portal 25 00000100:00000001:2.0:1784949326.260608:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949326.260611:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554960640 : -115544154590976 : ffff96e9c7b5b100) 00000100:00000001:3.0:1784949326.260617:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949326.260625:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949326.260630:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11603) 00000100:00000001:2.0:1784949326.260644:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949326.260647:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949326.260653:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949326.260655:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949326.260658:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000400:00000010:2.0:1784949326.260660:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000100:00000001:3.0:1784949326.260662:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949326.260670:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949326.260672:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289689728, offset 0 00000100:00000001:0.0:1784949326.260684:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1784949326.260686:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949326.260689:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949326.260695:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949326.260698:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949326.260702:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949326.260713:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:0.0:1784949326.260715:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949326.260735:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949326.260740:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949326.260741:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ee80 02000000:00000001:0.0:1784949326.260746:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949326.260751:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949326.260758:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8cb5 [2] + 0 02000000:00000001:0.0:1784949326.260760:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949326.260771:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000df521cee. 00000100:00100000:0.0:1784949326.260772:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289689856:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949326.260791:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1784949326.260793:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bfafda61 02000000:00000001:0.0:1784949326.260796:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949326.260800:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949326.260803:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1784949326.260806:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:0.0:1784949326.260810:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000208850ab. 00000400:00000200:2.0:1784949326.260818:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:0.0:1784949326.260819:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949326.260826:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bfafda61 00000400:00000010:0.0:1784949326.260828:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c7491352. 00000400:00000010:2.0:1784949326.260831:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bfafda61. 00000100:00000001:2.0:1784949326.260840:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949326.260843:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949326.260847:0:337120:0:(events.c:337:request_in_callback()) incoming req@0000000012145ed7 x1871651289689728 msgsize 224 00000400:00000010:0.0:1784949326.260853:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00100000:2.0:1784949326.260855:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949326.260864:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000200:0.0:1784949326.260865:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289689856, portal 10 00000100:00000001:0.0:1784949326.260872:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949326.260875:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000100:00000040:0.0:1784949326.260884:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949326.260887:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949326.260900:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000100:00000001:0.0:1784949326.260942:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949326.260949:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000010:2.0:1784949326.260950:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000100:00000040:0.0:1784949326.260955:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00100000:3.0:1784949326.260957:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289689728 00000100:00000001:2.0:1784949326.260961:0:337120:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1784949326.260965:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000010:0.0:1784949326.260967:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007749b1b7. 00000100:00000001:3.0:1784949326.260971:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000200:2.0:1784949326.260972:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949326.260979:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949326.260983:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289689856, offset 0 02000000:00000001:3.0:1784949326.260990:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949326.261002:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289689728 00000400:00000200:0.0:1784949326.261008:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:3.0:1784949326.261010:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949326.261016:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1784949326.261017:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784949326.261023:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:2.0:1784949326.261024:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1784949326.261029:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000400:00000200:0.0:1784949326.261030:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:3.0:1784949326.261041:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:0.0:1784949326.261051:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1784949326.261053:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949326.261068:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949326.261070:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949326.261074:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:2.0:1784949326.261079:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1784949326.261082:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000493fc253. 00000400:00000200:0.0:1784949326.261085:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1784949326.261091:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949326.261097:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:3.0:1784949326.261100:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000493fc253. 00000400:00000200:0.0:1784949326.261108:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3883:3883:246:4294967295] 192.168.202.137@tcp LPNI seq info [2490:2490:8:4294967295] 00000020:00000010:3.0:1784949326.261112:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000072ce5ef0. 00000020:00000010:3.0:1784949326.261122:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b74af153. 00000400:00000200:0.0:1784949326.261133:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:3.0:1784949326.261134:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000800:00000200:0.0:1784949326.261151:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949326.261154:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949326.261157:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000800:00000010:0.0:1784949326.261159:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008f5595bf. 00000100:00000001:2.0:1784949326.261163:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306265643 00000100:00000001:3.0:1784949326.261165:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949326.261172:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000200:0.0:1784949326.261176:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000010:2.0:1784949326.261177:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000082553b41. 00000100:00000001:3.0:1784949326.261179:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.261195:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949326.261197:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949326.261199:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949326.261209:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000008f5595bf type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949326.261214:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306265643 00000100:00000001:3.0:1784949326.261221:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784949326.261222:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000445487c0. 00000100:00000001:0.0:1784949326.261230:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.261234:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1784949326.261239:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2252 usecs (3 jiffies) 00000100:00000001:0.0:1784949326.261270:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306265604 00000100:00000001:0.0:1784949326.261275:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306265643 00000100:00000001:3.0:1784949326.261276:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949326.261278:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306265643 00000100:00000001:0.0:1784949326.261294:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306265604 00000100:00000001:3.0:1784949326.261297:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000010:2.0:1784949326.261299:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f4af3cce. 00000100:00000001:0.0:1784949326.261302:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306265643 00000100:00000001:2.0:1784949326.261311:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00100000:3.0:1784949326.261312:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 828 00000100:00000001:0.0:1784949326.261314:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306265604 00000100:00000040:3.0:1784949326.261327:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949326.261328:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306265643 00000100:00000001:2.0:1784949326.261328:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306265643 00000100:00000001:3.0:1784949326.261336:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180539472 : -115539529012144 : ffff96eadb6a6c50) 00000100:00000001:0.0:1784949326.261342:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306265604 00000100:00000010:2.0:1784949326.261344:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000abbd7d53. 00000100:00000040:3.0:1784949326.261351:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000012145ed7 x1871651289689728/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:232/0 lens 224/0 e 0 to 0 dl 1784949337 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949326.261356:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949326.261366:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949326.261368:0:337122:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306265643 00000100:00000001:3.0:1784949326.261373:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949326.261376:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949326.261378:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949326.261382:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000493fc253. 00000100:00000001:2.0:1784949326.261384:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949326.261386:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949326.261391:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949326.261395:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949326.261398:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949326.261400:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000493fc253. 00000100:00000001:2.0:1784949326.261407:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949326.261411:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.261413:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1784949326.261421:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000012145ed7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289689728:12345-0@lo:400: 00000100:00000001:2.0:1784949326.261425:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1784949326.261434:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289689728 00000100:00000001:2.0:1784949326.261435:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949326.261442:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949326.261443:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949326.261449:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949326.261451:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949326.261457:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949326.261462:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949326.261465:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:3.0:1784949326.261470:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949326.261473:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949326.261476:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949326.261479:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949326.261482:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949326.261486:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949326.261490:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949326.261495:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949326.261502:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000cad84312. 00000800:00000001:2.0:1784949326.261503:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949326.261509:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949326.261513:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949326.261515:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949326.261534:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784949326.261535:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008f5595bf. 00000020:00000001:3.0:1784949326.261535:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949326.261547:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949326.261549:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000040:3.0:1784949326.261549:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289689728 00000800:00000001:2.0:1784949326.261555:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949326.261562:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:1.0:1784949326.261573:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000200:3.0:1784949326.261574:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000012145ed7 x1871651289689728/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:232/0 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949326.261581:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949326.261587:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007749b1b7 00000400:00000010:1.0:1784949326.261595:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007749b1b7. 00010000:00000001:3.0:1784949326.261607:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949326.261614:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949326.261615:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949326.261619:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ef00 00000100:00001000:3.0:1784949326.261625:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=48 v=5 (1 1 1 1) 00000100:00000200:1.0:1784949326.261629:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949326.261642:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 181136 00000100:00000001:3.0:1784949326.261643:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949326.261654:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000001:2.0:1784949326.261663:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949326.261666:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:2.0:1784949326.261675:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949326.261678:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784949326.261680:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949326.261682:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784949326.261684:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949326.261692:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949326.261694:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784949326.261698:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000040:3.0:1784949326.261709:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949326.261716:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000400:00000010:3.0:1784949326.261720:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000200:2.0:1784949326.261723:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784949326.261728:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949326.261733:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289689728, offset 224 00000100:00000001:1.0:1784949326.261734:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:2.0:1784949326.261734:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001c5bdf08. 00000100:00000040:2.0:1784949326.261746:0:337114:0:(events.c:337:request_in_callback()) incoming req@000000001c5bdf08 x1871651289689856 msgsize 224 00000400:00000200:3.0:1784949326.261757:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:2.0:1784949326.261768:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:3.0:1784949326.261789:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:2.0:1784949326.261797:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949326.261809:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ee80 00000800:00000001:2.0:1784949326.261813:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949326.261824:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949326.261824:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x964d [1] + 224 00000100:00000001:1.0:1784949326.261834:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:3.0:1784949326.261841:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:1.0:1784949326.261843:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289689856 02000000:00000001:1.0:1784949326.261849:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949326.261856:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:3.0:1784949326.261858:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949326.261863:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949326.261871:0:342756:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1784949326.261872:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949326.261880:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289689856 00000100:00000200:3.0:1784949326.261883:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949326.261886:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949326.261891:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949326.261897:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949326.261936:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949326.261942:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949326.261946:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949326.261950:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949326.261956:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000100:00000001:1.0:1784949326.261960:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949326.261963:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000020:00000010:1.0:1784949326.261970:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000bbdbdf4. 00000100:00000040:3.0:1784949326.261972:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949326.261979:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000b1b47c1. 00000020:00000010:1.0:1784949326.261990:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949326.262002:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:3.0:1784949326.262004:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949326.262006:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949326.262011:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949326.262014:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949326.262016:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:3.0:1784949326.262022:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:1.0:1784949326.262023:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949326.262031:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949326.262032:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949326.262035:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949326.262038:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949326.262047:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949326.262051:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:3.0:1784949326.262055:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:2.0:1784949326.262058:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949326.262067:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949326.262072:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1784949326.262073:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949326.262082:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949326.262088:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000001:2.0:1784949326.262089:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1784949326.262090:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000012145ed7 x1871651289689728/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:232/0 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949326.262096:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000001:2.0:1784949326.262098:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949326.262106:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bcdfbf21. 00000100:00100000:1.0:1784949326.262115:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 804 00000100:00000001:0.0:1784949326.262119:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:2.0:1784949326.262120:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000100:00000001:0.0:1784949326.262128:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:2.0:1784949326.262130:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000100:00100000:3.0:1784949326.262131:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000012145ed7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289689728:12345-0@lo:400: Request processed in 753us (1277us total) trans 0 rc 0/0 00000100:00000040:1.0:1784949326.262132:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949326.262135:0:343544:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.262142:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784949326.262143:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469576448 : -115544239975168 : ffff96e9c29ed500) 00000100:00000001:0.0:1784949326.262145:0:343544:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949326.262156:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949326.262161:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000001c5bdf08 x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:232/0 lens 224/0 e 0 to 0 dl 1784949337 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949326.262172:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 828 00000100:00000040:3.0:1784949326.262184:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949326.262194:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949326.262200:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949326.262209:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b74af153. 00000020:00000010:3.0:1784949326.262222:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949326.262225:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000200:2.0:1784949326.262228:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949326.262233:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000072ce5ef0. 00000020:00000010:1.0:1784949326.262236:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000014ddbad0. 00000020:00000040:3.0:1784949326.262271:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949326.262272:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949326.262295:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000010:1.0:1784949326.262299:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000014ddbad0. 00000100:00000001:2.0:1784949326.262299:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:3.0:1784949326.262303:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000cad84312. 00000100:00000001:2.0:1784949326.262305:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949326.262315:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949326.262317:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00100000:1.0:1784949326.262319:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000001c5bdf08 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289689856:12345-192.168.202.137@tcp:400: 02000000:00000001:2.0:1784949326.262321:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949326.262324:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949326.262328:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949326.262336:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949326.262342:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289689856 00000100:00000200:3.0:1784949326.262342:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 02000000:00000001:2.0:1784949326.262343:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.262353:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00001000:2.0:1784949326.262357:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=48 v=5 (5 5 5 5) 00000400:00000010:3.0:1784949326.262358:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fed0cd52. 00000020:00000001:1.0:1784949326.262362:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949326.262370:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949326.262372:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=48 v=5 (1 1 1 1) 00000400:00000010:3.0:1784949326.262376:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003e1c50f6. 00000020:00000001:1.0:1784949326.262381:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:2.0:1784949326.262388:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:1.0:1784949326.262390:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784949326.262394:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.262397:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00010000:00000001:2.0:1784949326.262402:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:1.0:1784949326.262404:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949326.262407:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.262410:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949326.262416:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:2.0:1784949326.262416:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.262422:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.262427:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:2.0:1784949326.262428:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949326.262433:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949326.262439:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949326.262446:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1784949326.262447:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007f0bf97c. 00000100:00000001:2.0:1784949326.262453:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1784949326.262455:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.262460:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949326.262462:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.262468:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949326.262471:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1784949326.262473:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289689856 00010000:00000001:1.0:1784949326.262480:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949326.262491:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000001c5bdf08 x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:232/0 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949326.262499:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000fbc2d61e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289689728:0@lo:400: 00000100:00000001:2.0:1784949326.262517:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949326.262521:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949326.262524:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949326.262530:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000fbc2d61e x1871651289689728/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949326.262535:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949326.262544:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=43 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949326.262547:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1784949326.262552:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949326.262554:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949326.262559:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000b1cbf66f. 00000100:00000040:1.0:1784949326.262560:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949326.262569:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949326.262571:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1784949326.262574:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:2.0:1784949326.262577:0:337120:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:1.0:1784949326.262578:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949326.262582:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949326.262585:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1784949326.262589:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:1.0:1784949326.262595:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 02000000:00000010:2.0:1784949326.262596:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003547cd5f. 00000400:00000010:1.0:1784949326.262601:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 02000000:00000001:2.0:1784949326.262605:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000200:1.0:1784949326.262609:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289689856, offset 224 02000000:00000001:2.0:1784949326.262623:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:1.0:1784949326.262624:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000010:2.0:1784949326.262630:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000fbc2d61e. 00000100:00000001:2.0:1784949326.262637:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949326.262639:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949326.262646:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784949326.262647:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949326.262661:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784949326.262662:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3884:3884:246:4294967295] 192.168.202.137@tcp LPNI seq info [2491:2491:8:4294967295] 00000100:00000001:2.0:1784949326.262666:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949326.262675:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949326.262682:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949326.262690:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949326.262698:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949326.262703:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007a3f8609. 00000800:00000200:1.0:1784949326.262710:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949326.262716:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949326.262720:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007a3f8609 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949326.262740:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949326.262743:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949326.262747:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949326.262748:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949326.262751:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949326.262755:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000001c5bdf08 x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:232/0 lens 224/224 e 0 to 0 dl 1784949337 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949326.262788:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000001c5bdf08 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289689856:12345-192.168.202.137@tcp:400: Request processed in 564us (1040us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949326.262798:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 804 00000100:00000040:1.0:1784949326.262801:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949326.262804:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949326.262806:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949326.262809:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000b1b47c1. 00000020:00000010:1.0:1784949326.262814:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949326.262817:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000bbdbdf4. 00000020:00000040:1.0:1784949326.262822:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949326.262833:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949326.262997:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949326.263004:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007a3f8609. 00000400:00000200:3.0:1784949326.263012:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949326.263019:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949326.263023:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949326.263032:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000800:00000001:1.0:1784949326.263033:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1784949326.263035:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000800:00000001:1.0:1784949326.263041:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949326.263044:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:1.0:1784949326.263047:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949326.263049:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949326.263067:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949326.263081:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ef00 00000400:00000200:1.0:1784949326.263090:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9655 [1] + 224 00000800:00000001:1.0:1784949326.263097:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949326.263111:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949326.263113:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949326.263118:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949326.263123:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949326.263127:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949326.263139:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949326.263159:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949326.263167:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949326.263170:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949326.263300:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949326.263306:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949326.263313:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949326.263314:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949326.263317:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c7491352. 00000400:00000200:0.0:1784949326.263324:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:0.0:1784949326.263326:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:0.0:1784949326.263330:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949326.263338:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949326.263351:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949326.263361:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949326.263363:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949326.263365:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949326.263368:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949326.263370:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949326.263372:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949326.263374:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949326.263380:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=43 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949326.263384:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=43 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949326.263387:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949326.263389:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949326.263391:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949326.263395:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949326.263404:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949326.263407:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949326.263409:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949326.263411:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949326.263413:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949326.263416:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949326.263424:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949326.263428:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949326.263430:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949326.263434:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949326.263443:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289689856:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949326.263447:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949326.263449:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949326.263452:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289689856/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949326.263459:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949326.263461:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949326.263463:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000208850ab. 02000000:00000001:0.0:1784949326.263468:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949326.263470:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949326.263471:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949326.263474:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949326.263476:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000a929270c. 02000000:00000001:0.0:1784949326.263480:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949326.263482:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949326.263483:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000100:00000001:0.0:1784949326.263498:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949326.263499:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949326.263501:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949326.263506:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949326.263508:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949326.263511:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949326.263513:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949326.304016:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306265643, at 4306265644 00000100:00000040:2.0:1784949326.304030:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306265643, at 4306265644 00000100:00000010:1.0:1784949326.304033:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000445487c0. 00000100:00000040:3.0:1784949326.304034:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306265643, at 4306265644 00000100:00000040:0.0:1784949326.304037:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306265643, at 4306265644 00000100:00000010:2.0:1784949326.304046:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f4af3cce. 00000100:00000010:3.0:1784949326.304050:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000abbd7d53. 00000100:00000010:0.0:1784949326.304053:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000082553b41. 00000800:00000100:3.0:1784949326.452017:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:3.0:1784949326.452023:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b9b32818. 00000400:00000200:3.0:1784949326.452032:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949326.452043:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949326.452060:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949326.452067:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:3.0:1784949326.452075:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000800:00000100:3.0:1784949326.452110:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.2@tcp 00000800:00000010:3.0:1784949326.452115:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000be1f3156. 00000400:00000200:3.0:1784949326.452122:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949326.452123:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949326.452132:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949326.452139:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000200:3.0:1784949326.452141:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11599 health :800 00000400:00000001:2.0:1784949326.452153:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949326.452158:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949326.452160:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000079974bb9 00000400:00000010:2.0:1784949326.452163:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000079974bb9. 00000400:00000200:2.0:1784949326.452170:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949326.452172:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949326.452179:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:3.0:1784949326.452181:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000010:2.0:1784949326.452190:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000467d4d37. 00000400:00000200:2.0:1784949326.452200:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:3.0:1784949326.452206:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:2.0:1784949326.452211:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:2.0:1784949326.452223:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000400:00000200:2.0:1784949326.452233:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000001:2.0:1784949326.452237:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949326.452239:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949326.452241:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f0c9b918 00000400:00000010:2.0:1784949326.452270:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f0c9b918. 00000400:00000200:2.0:1784949326.452276:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949326.452280:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:2.0:1784949326.452294:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c49404b8. 00000400:00000200:2.0:1784949326.452299:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-110 00000400:00000200:2.0:1784949326.452302:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -110 00000400:00000200:2.0:1784949326.452305:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00010000:00000001:2.0:1784949327.091997:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949327.092004:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949327.092011:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949327.092012:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949327.092090:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949327.092097:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949327.092102:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949327.284012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949327.284026:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949327.284028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949327.284033:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949327.284036:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949327.284040:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949327.284043:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949327.476013:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949327.476025:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000800:00000100:3.0:1784949327.476034:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.30@tcp 00000400:00000010:1.0:1784949327.476039:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000bbdbdf4. 00000800:00000010:3.0:1784949327.476041:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cee99234. 00000400:00000010:1.0:1784949327.476051:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000200:3.0:1784949327.476053:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949327.476063:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.30@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784949327.476065:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000a5d96d2. 00000400:00000200:3.0:1784949327.476074:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11600 health :700 00000400:00000200:1.0:1784949327.476076:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949327.476087:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949327.476094:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.30@tcp: 1 00000400:00000200:1.0:1784949327.476100:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949327.476107:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949327.476111:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3885:3885:248:4294967295] 192.168.202.137@tcp LPNI seq info [2492:2492:8:4294967295] 00000400:00000200:1.0:1784949327.476118:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949327.476125:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949327.476130:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949327.476134:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.30@tcp: -114 00000800:00000010:1.0:1784949327.476154:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a21d8ef3. 00000800:00000200:1.0:1784949327.476167:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949327.476179:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949327.476185:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a21d8ef3 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949327.476213:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000100:3.0:1784949327.476224:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000800:00000010:3.0:1784949327.476230:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000600b3b47. 00000400:00000200:2.0:1784949327.476231:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.30@tcp(00000000bf7eadde) state 0x840 00000400:00000200:3.0:1784949327.476260:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000001:2.0:1784949327.476267:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.476274:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1784949327.476277:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949327.476281:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b039801a 00000400:00000010:2.0:1784949327.476287:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b039801a. 00000400:00000200:2.0:1784949327.476312:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949327.476313:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949327.476319:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.30@tcp: -114 00000400:00000200:3.0:1784949327.476321:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000010:2.0:1784949327.476329:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000eb983bef. 00000400:00000200:2.0:1784949327.476337:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.30@tcp:-110 00000400:00000200:3.0:1784949327.476340:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:2.0:1784949327.476346:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.30@tcp(00000000bf7eadde) state 0x2040 rc -110 00000400:00000200:2.0:1784949327.476355:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.30@tcp 00000400:00000200:2.0:1784949327.476377:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:2.0:1784949327.476381:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.476382:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.476384:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:2.0:1784949327.476385:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000400:00000200:2.0:1784949327.476390:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.476394:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:2.0:1784949327.476398:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004f865a97. 00000400:00000200:2.0:1784949327.476404:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:2.0:1784949327.476407:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:2.0:1784949327.476411:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00000400:00000200:1.0:1784949327.476429:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.30@tcp NID 111.222.173.30@tcp: -110. discovery complete 00000400:00000200:1.0:1784949327.476437:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.173.30@tcp primary NID 111.222.173.30@tcp rc -110 00000800:00000200:0.0:1784949327.476448:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949327.476454:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a21d8ef3. 00000100:00000010:1.0:1784949327.476456:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 0000000014ddbad0. 00000400:00000200:0.0:1784949327.476462:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949327.476467:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949327.476473:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949327.476479:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949327.476485:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949327.476485:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949327.476486:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000800:00000001:3.0:1784949327.476494:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:1.0:1784949327.476498:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=0000000014ddbad0 refcount 1 to 111.222.173.30@tcp 00000800:00000001:3.0:1784949327.476522:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000010:1.0:1784949327.476524:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000442a8405. 00000400:00000200:3.0:1784949327.476552:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949327.476588:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949327.476617:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949327.476650:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949327.476671:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3886:3886:250:4294967295] 192.168.202.137@tcp LPNI seq info [2493:2493:8:4294967295] 00000400:00000200:3.0:1784949327.476722:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00010000:00080000:1.0:1784949327.476734:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.173.30@tcp at tail 00010000:00000001:1.0:1784949327.476744:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949327.476745:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:1.0:1784949327.476755:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949327.476757:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949327.476767:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000010:1.0:1784949327.476777:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 00000000a41d8ff9. 00000800:00000200:3.0:1784949327.476785:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000400:1.0:1784949327.476794:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.142.30@tcp: rc = -75 00000800:00000200:3.0:1784949327.476794:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 3, nob 144 niov 1 nkiov 1 00000100:00000040:1.0:1784949327.476804:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.142.30@tcp cnt=32 00000400:00000200:1.0:1784949327.476810:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:1.0:1784949327.476813:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:1.0:1784949327.476817:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:1.0:1784949327.476819:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:1.0:1784949327.476822:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:1.0:1784949327.476829:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1784949327.476860:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.476865:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.476869:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:1.0:1784949327.476871:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:1.0:1784949327.476875:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949327.476893:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:1.0:1784949327.476896:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.476900:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000001:0.0:1784949327.476902:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949327.476937:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.476943:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:1.0:1784949327.476946:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:1.0:1784949327.476950:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949327.476958:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:1.0:1784949327.476960:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.476965:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.476969:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.476972:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000800:00000200:3.0:1784949327.476973:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1784949327.476974:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000800:00000010:3.0:1784949327.476982:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:1.0:1784949327.476987:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.476989:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949327.476990:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949327.476994:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949327.476998:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000800:00000001:0.0:1784949327.477001:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949327.477004:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949327.477006:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949327.477012:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949327.477013:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949327.477018:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000800:00000001:0.0:1784949327.477020:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949327.477027:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949327.477027:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477035:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000010:2.0:1784949327.477041:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000200:1.0:1784949327.477043:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:0.0:1784949327.477046:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000010:2.0:1784949327.477050:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e6156599. 00000400:00000200:1.0:1784949327.477055:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949327.477063:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a6b6402e. 00000400:00000200:1.0:1784949327.477071:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:0.0:1784949327.477075:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9669 00000400:00000200:1.0:1784949327.477077:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477082:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000001:0.0:1784949327.477087:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949327.477094:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:2.0:1784949327.477095:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000800:00000001:0.0:1784949327.477100:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949327.477104:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000200:0.0:1784949327.477105:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949327.477111:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:2.0:1784949327.477113:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:0.0:1784949327.477118:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:1.0:1784949327.477123:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477124:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.477151:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.477155:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.477159:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:2.0:1784949327.477163:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3887:3887:250:4294967295] 111.222.173.1@tcp LPNI seq info [55:55:8:4294967295] 00000400:00000200:2.0:1784949327.477170:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:2.0:1784949327.477177:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.477184:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000800:00000010:2.0:1784949327.477191:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004c284093. 00000400:00000200:0.0:1784949327.477192:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cdea04e8 00000400:00000200:1.0:1784949327.477196:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000010:0.0:1784949327.477201:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000cdea04e8. 00000400:00000200:1.0:1784949327.477202:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477212:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949327.477213:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:0.0:1784949327.477214:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000200:1.0:1784949327.477226:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477262:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000800:00000200:2.1:1784949327.477281:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000010:0.0:1784949327.477282:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000200:1.0:1784949327.477283:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000010:0.0:1784949327.477291:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000bbdbdf4. 00000400:00000200:1.0:1784949327.477296:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477297:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000800:00000001:0.0:1784949327.477304:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949327.477308:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:1.0:1784949327.477311:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000800:00000001:0.0:1784949327.477312:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949327.477318:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477325:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949327.477329:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:1.0:1784949327.477335:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477340:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:1.0:1784949327.477342:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000010:2.0:1784949327.477344:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef8e57c8. 00000400:00000200:1.0:1784949327.477346:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949327.477353:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000aaaf29ff. 00000400:00000200:1.0:1784949327.477357:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000400:00000200:1.0:1784949327.477360:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949327.477364:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000046ae30fa. 00000400:00000200:1.0:1784949327.477365:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.477371:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477376:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:2.0:1784949327.477377:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:1.0:1784949327.477381:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000400:00000200:1.0:1784949327.477386:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477391:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:2.0:1784949327.477395:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.477397:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.477400:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.477402:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:2.0:1784949327.477405:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3888:3888:249:4294967295] 111.222.142.1@tcp LPNI seq info [53:53:8:4294967295] 00000400:00000200:2.0:1784949327.477409:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:2.0:1784949327.477415:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.477419:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000800:00000010:2.0:1784949327.477423:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000031deb17e. 00000400:00000200:1.0:1784949327.477424:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:1.0:1784949327.477429:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477431:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949327.477436:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:1.0:1784949327.477439:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477445:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000800:00000200:2.1:1784949327.477450:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:1.0:1784949327.477452:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000200:1.0:1784949327.477460:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949327.477463:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000200:1.0:1784949327.477465:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477466:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.477468:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477469:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:1.0:1784949327.477471:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:2.0:1784949327.477476:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:1.0:1784949327.477477:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477486:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:1.0:1784949327.477489:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: 0 00000400:00000200:1.0:1784949327.477494:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477498:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949327.477503:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:1.0:1784949327.477510:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477514:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000200:1.0:1784949327.477516:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000400:00000010:2.0:1784949327.477534:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000200:1.0:1784949327.477540:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949327.477544:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000400:00000200:1.0:1784949327.477551:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000400:00000200:1.0:1784949327.477555:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949327.477560:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000074335e5d. 00000400:00000200:1.0:1784949327.477561:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.477572:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:2.0:1784949327.477580:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:1.0:1784949327.477582:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:1.0:1784949327.477588:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:2.0:1784949327.477595:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:1.0:1784949327.477597:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477609:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.477613:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.477616:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.477620:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:2.0:1784949327.477625:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3889:3889:248:4294967295] 111.222.173.2@tcp LPNI seq info [52:52:8:4294967295] 00000400:00000200:2.0:1784949327.477632:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:2.0:1784949327.477639:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.477646:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:2.0:1784949327.477650:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000400:00000200:1.0:1784949327.477651:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:1.0:1784949327.477659:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477664:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949327.477667:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:1.0:1784949327.477674:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000800:00000200:2.0:1784949327.477677:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000400:00000200:1.0:1784949327.477680:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:1.0:1784949327.477689:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000800:00000200:2.1:1784949327.477695:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:1.0:1784949327.477700:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949327.477703:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.2 (cnted 0, interval 60, 12 secs later) 00000400:00000200:1.0:1784949327.477710:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:1.0:1784949327.477716:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000100:2.0:1784949327.477718:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.2@tcp 00000400:00000200:1.0:1784949327.477721:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000010:2.0:1784949327.477724:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:1.0:1784949327.477733:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:2.0:1784949327.477737:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949327.477739:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000400:00000200:1.0:1784949327.477746:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:2.0:1784949327.477754:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.2@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949327.477761:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477763:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949327.477774:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.2@tcp: 1 00000400:00000200:1.0:1784949327.477778:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:1.0:1784949327.477783:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949327.477792:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000200:1.0:1784949327.477803:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.477810:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477813:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:1.0:1784949327.477816:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:2.0:1784949327.477824:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:1.0:1784949327.477825:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477854:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 rc 0 00000400:00000200:1.0:1784949327.477857:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:1.0:1784949327.477861:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.477866:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949327.477870:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:1.0:1784949327.477877:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477880:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:1.0:1784949327.477882:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000010:2.0:1784949327.477884:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000047e19e91. 00000400:00000200:1.0:1784949327.477886:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949327.477892:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000309fc37c. 00000400:00000200:1.0:1784949327.477895:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000400:00000200:1.0:1784949327.477899:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:2.0:1784949327.477903:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009535813f. 00000400:00000200:1.0:1784949327.477904:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.477911:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.477917:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:2.0:1784949327.477918:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:2.0:1784949327.477927:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:2.0:1784949327.477930:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949327.477943:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:1.0:1784949327.477949:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.477961:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.477968:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.477971:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:2.0:1784949327.477975:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3890:3890:248:4294967295] 111.222.142.2@tcp LPNI seq info [52:52:8:4294967295] 00000400:00000200:2.0:1784949327.477981:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:2.0:1784949327.477990:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.478001:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000400:00000200:1.0:1784949327.478005:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:1.0:1784949327.478009:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784949327.478009:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ffbbc320. 00000400:00000200:1.0:1784949327.478016:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949327.478021:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:1.0:1784949327.478027:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478031:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:1.0:1784949327.478033:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:1.0:1784949327.478040:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949327.478040:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:2.0:1784949327.478051:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:1.0:1784949327.478053:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:1.0:1784949327.478057:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1784949327.478061:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x261 rc 0 00000400:00000200:1.0:1784949327.478062:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478069:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478073:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000400:00000200:1.0:1784949327.478075:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000400:00000200:2.0:1784949327.478079:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:1.0:1784949327.478084:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949327.478088:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002c1a3d41. 00000400:00000200:1.0:1784949327.478093:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000400:00000010:2.0:1784949327.478095:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007c185147. 00000400:00000200:1.0:1784949327.478096:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478102:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478105:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478107:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000010:2.0:1784949327.478108:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000f7452f34. 00000400:00000200:1.0:1784949327.478110:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:1.0:1784949327.478122:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.478123:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:1.0:1784949327.478150:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:1.0:1784949327.478152:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478156:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478161:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478165:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:1.0:1784949327.478168:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:2.0:1784949327.478174:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:1.0:1784949327.478178:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.478184:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.478189:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.478193:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.478199:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:2.0:1784949327.478203:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3891:3891:247:4294967295] 111.222.173.3@tcp LPNI seq info [50:50:8:4294967295] 00000400:00000200:2.0:1784949327.478209:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:2.0:1784949327.478216:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.478222:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:2.0:1784949327.478227:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c2b2d4f6. 00000400:00000200:1.0:1784949327.478228:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: 0 00000400:00000200:1.0:1784949327.478251:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478254:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478257:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478260:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000800:00000200:2.0:1784949327.478267:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:1.0:1784949327.478291:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000200:1.0:1784949327.478303:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949327.478304:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:1.0:1784949327.478310:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:2.0:1784949327.478314:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:1.0:1784949327.478315:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478321:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949327.478321:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:1.0:1784949327.478328:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478331:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:1.0:1784949327.478333:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:2.0:1784949327.478337:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:1.0:1784949327.478342:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949327.478346:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000032fe57fd. 00000400:00000200:1.0:1784949327.478350:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000010:2.0:1784949327.478353:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000200:1.0:1784949327.478354:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478362:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478364:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478366:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000010:2.0:1784949327.478367:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000625577d2. 00000400:00000200:1.0:1784949327.478370:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000400:00000200:1.0:1784949327.478381:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.478384:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:1.0:1784949327.478389:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:1.0:1784949327.478391:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478394:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478396:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478398:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:1.0:1784949327.478400:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:2.0:1784949327.478404:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:1.0:1784949327.478405:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.478414:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.478418:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.478420:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.478422:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:2.0:1784949327.478425:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3892:3892:246:4294967295] 111.222.142.3@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:2.0:1784949327.478429:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:2.0:1784949327.478434:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.478438:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:2.0:1784949327.478441:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e79c6895. 00000400:00000200:1.0:1784949327.478442:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000200:1.0:1784949327.478448:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478450:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949327.478454:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:1.0:1784949327.478459:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478463:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000800:00000200:2.1:1784949327.478464:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:1.0:1784949327.478469:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000200:1.0:1784949327.478474:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949327.478477:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:1.0:1784949327.478479:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478482:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949327.478485:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:1.0:1784949327.478490:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478495:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:2.0:1784949327.478496:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x261 rc 0 00000400:00000200:1.0:1784949327.478502:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:2.0:1784949327.478508:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:1.0:1784949327.478511:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000010:2.0:1784949327.478517:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000200:1.0:1784949327.478521:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000010:2.0:1784949327.478542:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000492bb2ef. 00000400:00000200:1.0:1784949327.478543:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478549:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478555:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478558:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000400:00000010:2.0:1784949327.478559:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a9f111b0. 00000400:00000200:1.0:1784949327.478561:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:1.0:1784949327.478575:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.478578:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000400:00000200:1.0:1784949327.478587:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:1.0:1784949327.478590:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478593:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949327.478597:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478599:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:1.0:1784949327.478602:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:2.0:1784949327.478607:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00000400:00000200:1.0:1784949327.478613:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:2.0:1784949327.478621:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.478628:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.478632:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.478635:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:2.0:1784949327.478639:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3893:3893:245:4294967295] 111.222.173.4@tcp LPNI seq info [50:50:8:4294967295] 00000400:00000200:2.0:1784949327.478645:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:2.0:1784949327.478651:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.478661:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000400:00000200:1.0:1784949327.478667:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000800:00000010:2.0:1784949327.478670:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000400:00000200:1.0:1784949327.478672:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949327.478682:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:2.0:1784949327.478687:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:1.0:1784949327.478693:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000800:00000200:2.0:1784949327.478696:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000400:00000200:1.0:1784949327.478699:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000200:1.0:1784949327.478708:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000800:00000200:2.1:1784949327.478716:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:1.0:1784949327.478723:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000800:00000200:2.1:1784949327.478724:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.4 (cnted 0, interval 60, 12 secs later) 00000400:00000200:1.0:1784949327.478734:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000800:00000100:2.0:1784949327.478743:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.4@tcp 00000400:00000200:1.0:1784949327.478744:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000010:2.0:1784949327.478751:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:1.0:1784949327.478752:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1784949327.478762:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949327.478767:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:1.0:1784949327.478773:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000400:00000200:2.0:1784949327.478780:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.4@tcp: GET: REMOTE_ERROR 00000100:00000010:1.0:1784949327.478782:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 00000000a41d8ff9. 00000400:00000200:2.0:1784949327.478787:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000020:00000001:1.0:1784949327.478796:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000200:2.0:1784949327.478798:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.4@tcp: 1 00010000:00000001:1.0:1784949327.478805:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000400:00000200:2.0:1784949327.478811:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000100:00000001:1.0:1784949327.478812:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000010:1.0:1784949327.478845:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000229221cc. 00000400:00000200:2.0:1784949327.478845:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:2.0:1784949327.478852:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 rc 0 00000400:00000200:1.0:1784949327.478857:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000229221cc nid 111.222.142.30@tcp 00000400:00000200:2.0:1784949327.478863:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000010:1.0:1784949327.478865:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000007e14bd74. 00000400:00000200:1.0:1784949327.478870:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 000000007e14bd74 net tcp 00000400:00000010:2.0:1784949327.478871:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001df3b40e. 00000400:00000010:1.0:1784949327.478875:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000000bbdbdf4. 00000400:00000010:2.0:1784949327.478878:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000864ba3de. 00000400:00000200:1.0:1784949327.478884:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 000000000bbdbdf4 nid 111.222.142.30@tcp 00000400:00000010:2.0:1784949327.478884:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000aa2db199. 00000400:00000200:2.0:1784949327.478894:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:1.0:1784949327.478900:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.142.30@tcp NID 111.222.142.30@tcp flags 0x0 00000400:00000200:1.0:1784949327.478906:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.142.30@tcp: 0 00000400:00000200:2.0:1784949327.478911:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:2.0:1784949327.478917:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.478920:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.478924:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.478927:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:2.0:1784949327.478930:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3894:3894:245:4294967295] 111.222.142.4@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:2.0:1784949327.478961:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:2.0:1784949327.478974:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.478979:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:2.0:1784949327.478983:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000081367938. 00000400:00000200:1.0:1784949327.478989:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.30@tcp: 0 00000400:00000200:1.0:1784949327.478993:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000800:00000200:2.0:1784949327.478994:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:2.1:1784949327.479002:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000400:00000200:2.0:1784949327.479006:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:2.0:1784949327.479010:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x261 rc 0 00000400:00000200:2.0:1784949327.479014:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 00000400:00000010:2.0:1784949327.479019:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000003cdea007. 00000400:00000010:2.0:1784949327.479023:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006fb22a7b. 00000400:00000010:2.0:1784949327.479028:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001b175f1d. 00000400:00000200:2.0:1784949327.479033:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.5@tcp 00000400:00000200:2.0:1784949327.479040:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.5@tcp local destination 00000400:00000200:2.0:1784949327.479044:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479048:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479051:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479055:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.5@tcp 00000400:00000200:2.0:1784949327.479062:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3895:3895:244:4294967295] 111.222.142.5@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:2.0:1784949327.479067:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.5@tcp 00000400:00000200:2.0:1784949327.479078:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.5@tcp(111.222.142.5@tcp:111.222.142.5@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479085:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.5@tcp 00000800:00000010:2.0:1784949327.479090:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000061dbde97. 00000800:00000200:2.0:1784949327.479095:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000800:00000200:2.1:1784949327.479101:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000001c95687] -> 12345-111.222.142.5@tcp (2) 00000400:00000200:2.0:1784949327.479116:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.5@tcp 00000400:00000200:2.0:1784949327.479118:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x261 rc 0 00000400:00000200:2.0:1784949327.479123:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:2.0:1784949327.479133:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000008f67722c. 00000400:00000010:2.0:1784949327.479155:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000400:00000010:2.0:1784949327.479161:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000066c58e5b. 00000400:00000200:2.0:1784949327.479167:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:2.0:1784949327.479174:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:2.0:1784949327.479179:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479185:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479188:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479192:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:2.0:1784949327.479197:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3896:3896:243:4294967295] 111.222.173.6@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:2.0:1784949327.479205:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:2.0:1784949327.479217:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479223:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:2.0:1784949327.479229:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000012f4d1ec. 00000800:00000200:2.0:1784949327.479233:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.0:1784949327.479264:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784949327.479286:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:2.1:1784949327.479288:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.6 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784949327.479292:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.6@tcp 00000800:00000010:2.0:1784949327.479293:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000012f4d1ec. 00000400:00000200:2.0:1784949327.479297:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949327.479301:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.6@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949327.479304:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949327.479307:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.6@tcp: 1 00000400:00000200:2.0:1784949327.479311:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000200:2.0:1784949327.479317:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:2.0:1784949327.479318:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 rc 0 00000400:00000200:2.0:1784949327.479321:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:2.0:1784949327.479326:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000011003cc7. 00000400:00000010:2.0:1784949327.479329:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000031068cb7. 00000400:00000010:2.0:1784949327.479335:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000392709f0. 00000400:00000200:2.0:1784949327.479338:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:2.0:1784949327.479342:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:2.0:1784949327.479345:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479347:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479349:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479351:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:2.0:1784949327.479354:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3897:3897:243:4294967295] 111.222.173.7@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1784949327.479358:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:2.0:1784949327.479365:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479369:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:2.0:1784949327.479375:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000012f4d1ec. 00000800:00000200:2.0:1784949327.479379:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:2.1:1784949327.479384:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:2.0:1784949327.479389:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:2.0:1784949327.479392:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000400:00000200:2.0:1784949327.479395:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:2.0:1784949327.479400:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f11303f8. 00000400:00000010:2.0:1784949327.479403:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000010:2.0:1784949327.479407:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000007c27c1cc. 00000400:00000200:2.0:1784949327.479411:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:2.0:1784949327.479417:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:2.0:1784949327.479420:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479422:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479426:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479429:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:2.0:1784949327.479432:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3898:3898:242:4294967295] 111.222.142.7@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:2.0:1784949327.479436:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:2.0:1784949327.479445:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479448:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:2.0:1784949327.479452:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000030b7fbec. 00000800:00000200:2.0:1784949327.479457:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:2.1:1784949327.479463:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:2.0:1784949327.479467:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:2.0:1784949327.479469:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:2.0:1784949327.479472:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x861 00000400:00000001:2.0:1784949327.479475:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.479477:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.479479:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:2.0:1784949327.479480:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000400:00000200:2.0:1784949327.479485:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.479492:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: -114 00000400:00000010:2.0:1784949327.479496:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000400:00000200:2.0:1784949327.479527:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.2@tcp:-5 00000400:00000200:2.0:1784949327.479532:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 rc -5 00000400:00000200:2.0:1784949327.479537:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.2@tcp 00000400:00000200:2.0:1784949327.479541:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:2.0:1784949327.479547:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002fbb6924. 00000400:00000010:2.0:1784949327.479553:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000400:00000010:2.0:1784949327.479558:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000076c3e16a. 00000400:00000200:2.0:1784949327.479565:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:2.0:1784949327.479571:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:2.0:1784949327.479576:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479579:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479581:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479584:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:2.0:1784949327.479587:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3899:3899:241:4294967295] 111.222.173.8@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1784949327.479591:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:2.0:1784949327.479597:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479604:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784949327.479609:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000eddb6110. 00000800:00000200:2.0:1784949327.479614:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.0:1784949327.479617:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784949327.479622:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:2.1:1784949327.479638:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.8 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784949327.479645:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.8@tcp 00000800:00000010:2.0:1784949327.479649:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000eddb6110. 00000400:00000200:2.0:1784949327.479653:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949327.479658:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.8@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949327.479662:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949327.479665:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.8@tcp: 1 00000400:00000200:2.0:1784949327.479668:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000200:2.0:1784949327.479674:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:2.0:1784949327.479676:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 rc 0 00000400:00000200:2.0:1784949327.479680:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:2.0:1784949327.479685:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:2.0:1784949327.479689:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006f3c62ee. 00000400:00000010:2.0:1784949327.479693:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003e5660bf. 00000400:00000200:2.0:1784949327.479696:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:2.0:1784949327.479701:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:2.0:1784949327.479704:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479707:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479710:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479713:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:2.0:1784949327.479716:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3900:3900:241:4294967295] 111.222.142.8@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784949327.479720:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:2.0:1784949327.479726:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479730:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:2.0:1784949327.479733:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000eddb6110. 00000800:00000200:2.0:1784949327.479737:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:2.1:1784949327.479741:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000400:00000200:2.0:1784949327.479744:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:2.0:1784949327.479746:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x261 rc 0 00000400:00000200:2.0:1784949327.479749:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:2.0:1784949327.479753:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f6f633a2. 00000400:00000010:2.0:1784949327.479755:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eb2479c. 00000400:00000010:2.0:1784949327.479759:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000002096019c. 00000400:00000200:2.0:1784949327.479762:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:2.0:1784949327.479766:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:2.0:1784949327.479769:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479771:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479773:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479776:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:2.0:1784949327.479779:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3901:3901:240:4294967295] 111.222.142.9@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1784949327.479782:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:2.0:1784949327.479788:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479792:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:2.0:1784949327.479795:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000d74efe59. 00000800:00000200:2.0:1784949327.479800:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:2.1:1784949327.479804:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:2.0:1784949327.479820:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:2.0:1784949327.479822:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:2.0:1784949327.479844:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:2.0:1784949327.479847:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007c65ffe9. 00000400:00000010:2.0:1784949327.479849:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000400:00000010:2.0:1784949327.479852:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b585333b. 00000400:00000200:2.0:1784949327.479855:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:2.0:1784949327.479860:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:2.0:1784949327.479864:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479866:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479868:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479870:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:2.0:1784949327.479873:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3902:3902:239:4294967295] 111.222.173.10@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:2.0:1784949327.479876:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:2.0:1784949327.479880:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.479884:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784949327.479886:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000017e5e4f6. 00000800:00000200:2.0:1784949327.479890:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.0:1784949327.479891:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784949327.479895:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:2.1:1784949327.479897:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.10 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784949327.479902:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.10@tcp 00000800:00000010:2.0:1784949327.479903:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000017e5e4f6. 00000400:00000200:2.0:1784949327.479905:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949327.479911:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.10@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949327.479913:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949327.479916:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.10@tcp: 1 00000400:00000200:2.0:1784949327.479919:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000200:2.0:1784949327.479952:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:2.0:1784949327.479957:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 rc 0 00000400:00000200:2.0:1784949327.479962:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 00000400:00000010:2.0:1784949327.479967:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000cdf2ee22. 00000400:00000010:2.0:1784949327.479970:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000357876c0. 00000400:00000010:2.0:1784949327.479973:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000066bbcdd2. 00000400:00000200:2.0:1784949327.479976:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.11@tcp 00000400:00000200:2.0:1784949327.479981:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.11@tcp local destination 00000400:00000200:2.0:1784949327.479983:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.479985:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.479987:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.479990:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.11@tcp 00000400:00000200:2.0:1784949327.479993:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3903:3903:239:4294967295] 111.222.173.11@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:2.0:1784949327.479997:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.11@tcp 00000400:00000200:2.0:1784949327.480001:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.11@tcp(111.222.173.11@tcp:111.222.173.11@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480006:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.11@tcp 00000800:00000010:2.0:1784949327.480008:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000017e5e4f6. 00000800:00000200:2.0:1784949327.480013:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000800:00000200:2.1:1784949327.480019:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a4f2972b] -> 12345-111.222.173.11@tcp (2) 00000400:00000200:2.0:1784949327.480024:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.11@tcp 00000400:00000200:2.0:1784949327.480028:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x261 rc 0 00000400:00000200:2.0:1784949327.480033:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:2.0:1784949327.480036:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ead2c724. 00000400:00000010:2.0:1784949327.480041:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d6365940. 00000400:00000010:2.0:1784949327.480046:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000056f87a18. 00000400:00000200:2.0:1784949327.480049:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:2.0:1784949327.480053:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:2.0:1784949327.480055:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480058:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480060:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480062:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:2.0:1784949327.480065:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3904:3904:238:4294967295] 111.222.142.11@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784949327.480069:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:2.0:1784949327.480073:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480076:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:2.0:1784949327.480080:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000676e0274. 00000800:00000200:2.0:1784949327.480084:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:2.1:1784949327.480087:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:2.0:1784949327.480091:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:2.0:1784949327.480095:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:2.0:1784949327.480099:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:2.0:1784949327.480108:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000008fae13bf. 00000400:00000010:2.0:1784949327.480113:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000400:00000010:2.0:1784949327.480118:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c5dc43e1. 00000400:00000200:2.0:1784949327.480121:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:2.0:1784949327.480125:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:2.0:1784949327.480127:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480129:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480148:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480151:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:2.0:1784949327.480156:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3905:3905:237:4294967295] 111.222.173.12@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1784949327.480161:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:2.0:1784949327.480172:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480179:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784949327.480185:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000aba78344. 00000800:00000200:2.0:1784949327.480190:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.0:1784949327.480192:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784949327.480197:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:2.1:1784949327.480200:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.12 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784949327.480204:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.12@tcp 00000800:00000010:2.0:1784949327.480207:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000aba78344. 00000400:00000200:2.0:1784949327.480211:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949327.480215:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.12@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949327.480219:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949327.480222:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.12@tcp: 1 00000400:00000200:2.0:1784949327.480227:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000200:2.0:1784949327.480256:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:2.0:1784949327.480265:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 rc 0 00000400:00000200:2.0:1784949327.480271:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:2.0:1784949327.480276:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004041a48e. 00000400:00000010:2.0:1784949327.480299:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056dea15c. 00000400:00000010:2.0:1784949327.480303:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000089841831. 00000400:00000200:2.0:1784949327.480307:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:2.0:1784949327.480313:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:2.0:1784949327.480316:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480320:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480322:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480324:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:2.0:1784949327.480327:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3906:3906:237:4294967295] 111.222.142.12@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:2.0:1784949327.480331:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:2.0:1784949327.480341:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480347:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:2.0:1784949327.480351:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000aba78344. 00000800:00000200:2.0:1784949327.480356:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:2.1:1784949327.480360:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000400:00000200:2.0:1784949327.480362:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:2.0:1784949327.480364:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x261 rc 0 00000400:00000200:2.0:1784949327.480367:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:2.0:1784949327.480372:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000461737cb. 00000400:00000010:2.0:1784949327.480375:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000400:00000010:2.0:1784949327.480380:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009c9d7b8b. 00000400:00000200:2.0:1784949327.480383:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:2.0:1784949327.480387:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:2.0:1784949327.480389:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480391:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480393:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480395:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:2.0:1784949327.480398:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3907:3907:236:4294967295] 111.222.173.13@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:2.0:1784949327.480403:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:2.0:1784949327.480407:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480411:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:2.0:1784949327.480414:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ce8f840d. 00000800:00000200:2.0:1784949327.480417:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:2.1:1784949327.480421:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:2.0:1784949327.480423:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:2.0:1784949327.480425:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:2.0:1784949327.480427:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:2.0:1784949327.480430:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f0e4e38. 00000400:00000010:2.0:1784949327.480432:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3f5c273. 00000400:00000010:2.0:1784949327.480435:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000cfebe24. 00000400:00000200:2.0:1784949327.480437:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:2.0:1784949327.480440:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:2.0:1784949327.480442:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480444:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480445:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480447:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:2.0:1784949327.480450:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3908:3908:235:4294967295] 111.222.142.13@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:2.0:1784949327.480453:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:2.0:1784949327.480458:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480461:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:2.0:1784949327.480463:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000009662d9a. 00000800:00000200:2.0:1784949327.480466:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:2.1:1784949327.480469:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:2.0:1784949327.480479:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:2.0:1784949327.480481:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:2.0:1784949327.480483:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:2.0:1784949327.480487:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000da755428. 00000400:00000010:2.0:1784949327.480490:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000400:00000010:2.0:1784949327.480493:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bbb19812. 00000400:00000200:2.0:1784949327.480497:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:2.0:1784949327.480518:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:2.0:1784949327.480521:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480524:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480527:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480530:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:2.0:1784949327.480533:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3909:3909:234:4294967295] 111.222.173.14@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1784949327.480538:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:2.0:1784949327.480547:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480553:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784949327.480557:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f64c9b3a. 00000800:00000200:2.0:1784949327.480562:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.0:1784949327.480566:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.14 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1784949327.480572:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:2.1:1784949327.480575:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.173.14 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1784949327.480580:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.173.14@tcp 00000800:00000010:2.0:1784949327.480583:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f64c9b3a. 00000400:00000200:2.0:1784949327.480587:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949327.480590:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.14@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949327.480595:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949327.480599:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.14@tcp: 1 00000400:00000200:2.0:1784949327.480607:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000200:2.0:1784949327.480614:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:2.0:1784949327.480616:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 rc 0 00000400:00000200:2.0:1784949327.480620:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:2.0:1784949327.480626:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006841197d. 00000400:00000010:2.0:1784949327.480629:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000002566e96. 00000400:00000010:2.0:1784949327.480632:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000fe349a1e. 00000400:00000200:2.0:1784949327.480635:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:2.0:1784949327.480639:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:2.0:1784949327.480642:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480645:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480647:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480650:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:2.0:1784949327.480653:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3910:3910:234:4294967295] 111.222.173.15@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1784949327.480657:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:2.0:1784949327.480662:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480667:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:2.0:1784949327.480670:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f64c9b3a. 00000800:00000200:2.0:1784949327.480675:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:2.1:1784949327.480682:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:2.0:1784949327.480687:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:2.0:1784949327.480689:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:2.0:1784949327.480694:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:2.0:1784949327.480698:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000be26f34f. 00000400:00000010:2.0:1784949327.480701:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d53ff17e. 00000400:00000010:2.0:1784949327.480706:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000062fa023a. 00000400:00000200:2.0:1784949327.480709:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:2.0:1784949327.480713:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:2.0:1784949327.480715:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1784949327.480718:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480720:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1784949327.480722:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:2.0:1784949327.480726:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3911:3911:233:4294967295] 111.222.142.15@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:2.0:1784949327.480730:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:2.0:1784949327.480737:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480741:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:2.0:1784949327.480746:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000008e487d72. 00000800:00000200:2.0:1784949327.480751:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:2.1:1784949327.480755:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:2.0:1784949327.480759:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:2.0:1784949327.480761:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:2.0:1784949327.480764:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x861 00000400:00000001:2.0:1784949327.480767:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.480768:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.480770:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 00000400:00000010:2.0:1784949327.480771:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000400:00000200:2.0:1784949327.480774:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.480778:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: -114 00000400:00000010:2.0:1784949327.480781:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089e37819. 00000400:00000200:2.0:1784949327.480785:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.4@tcp:-5 00000400:00000200:2.0:1784949327.480790:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 rc -5 00000400:00000200:2.0:1784949327.480795:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.4@tcp 00000400:00000200:2.0:1784949327.480803:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.30@tcp(00000000229221cc) state 0x40 00000400:00000010:2.0:1784949327.480811:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000010:2.0:1784949327.480817:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000492bb2ef. 00000400:00000010:2.0:1784949327.480839:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c2354731. 00000400:00000200:2.0:1784949327.480843:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.30@tcp 00000400:00000200:2.0:1784949327.480846:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.142.30@tcp local destination 00000400:00000200:2.0:1784949327.480849:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949327.480852:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.142.30@tcp 00000400:00000200:2.0:1784949327.480856:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.142.30@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:2.0:1784949327.480860:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3912:3912:232:4294967295] 111.222.142.30@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1784949327.480866:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.30@tcp 00000400:00000200:2.0:1784949327.480872:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.30@tcp(111.222.142.30@tcp:111.222.142.30@tcp) : GET try# 0 00000800:00000200:2.0:1784949327.480876:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.30@tcp 00000800:00000010:2.0:1784949327.480878:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000e39f0ad5. 00000800:00000010:2.0:1784949327.480883:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 00000000995a12aa. 00000800:00000010:2.0:1784949327.480886:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000fba7d3a1. 00000800:00000200:2.0:1784949327.480890:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000995a12aa] -> 12345-111.222.142.30@tcp (2) 00000800:00000200:2.1:1784949327.480893:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000995a12aa] -> 12345-111.222.142.30@tcp (2) 00000400:00000200:2.0:1784949327.480898:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.30@tcp 00000400:00000200:2.0:1784949327.480901:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.30@tcp(00000000229221cc) state 0x240 rc 0 00000400:00000200:2.0:1784949327.480903:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x861 00000400:00000001:2.0:1784949327.480905:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.480906:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.480907:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:2.0:1784949327.480909:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000400:00000200:2.0:1784949327.480913:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.480917:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: -114 00000400:00000010:2.0:1784949327.480921:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000008f67722c. 00000400:00000200:2.0:1784949327.480924:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.6@tcp:-5 00000400:00000200:2.0:1784949327.480927:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 rc -5 00000400:00000200:2.0:1784949327.480931:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.6@tcp 00000400:00000200:2.0:1784949327.480968:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x861 00000400:00000001:2.0:1784949327.480972:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.480975:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.480977:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:2.0:1784949327.480979:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000400:00000200:2.0:1784949327.480983:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.480989:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: -114 00000400:00000010:2.0:1784949327.480992:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002fbb6924. 00000400:00000200:2.0:1784949327.480996:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.8@tcp:-5 00000400:00000200:2.0:1784949327.480999:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 rc -5 00000400:00000200:2.0:1784949327.481002:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.8@tcp 00000400:00000200:2.0:1784949327.481004:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x861 00000400:00000001:2.0:1784949327.481006:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.481007:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.481008:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:2.0:1784949327.481011:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:2.0:1784949327.481013:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.481016:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: -114 00000400:00000010:2.0:1784949327.481017:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000007c65ffe9. 00000400:00000200:2.0:1784949327.481021:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.10@tcp:-5 00000400:00000200:2.0:1784949327.481022:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 rc -5 00000400:00000200:2.0:1784949327.481025:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.10@tcp 00000400:00000200:2.0:1784949327.481031:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x861 00000400:00000001:2.0:1784949327.481033:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.481034:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.481035:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:2.0:1784949327.481036:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000400:00000200:2.0:1784949327.481039:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.481041:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: -114 00000400:00000010:2.0:1784949327.481043:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000008fae13bf. 00000400:00000200:2.0:1784949327.481047:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.12@tcp:-5 00000400:00000200:2.0:1784949327.481049:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 rc -5 00000400:00000200:2.0:1784949327.481051:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.12@tcp 00000400:00000200:2.0:1784949327.481053:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x861 00000400:00000001:2.0:1784949327.481054:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949327.481055:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949327.481056:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784949327.481058:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000400:00000200:2.0:1784949327.481059:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949327.481061:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: -114 00000400:00000010:2.0:1784949327.481062:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000da755428. 00000400:00000200:2.0:1784949327.481065:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.14@tcp:-5 00000400:00000200:2.0:1784949327.481067:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 rc -5 00000400:00000200:2.0:1784949327.481069:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.14@tcp 00010000:00000001:2.0:1784949328.115990:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949328.115997:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949328.116001:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949328.116003:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949328.116079:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949328.116082:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949328.116085:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949328.307985:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949328.307997:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949328.307999:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949328.308003:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949328.308005:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949328.308009:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949328.308011:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949328.500028:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.6@tcp 00000800:00000010:3.0:1784949328.500033:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b80ce7eb. 00000400:00000200:3.0:1784949328.500040:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949328.500045:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949328.500052:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949328.500056:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:3.0:1784949328.500059:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:1.0:1784949328.500090:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949328.500097:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000400:00000010:1.0:1784949328.500104:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000467d4d37. 00000400:00000200:2.0:1784949328.500108:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:1.0:1784949328.500109:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000400:00000010:1.0:1784949328.500118:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000063d06960. 00000400:00000200:2.0:1784949328.500119:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2861 00000400:00000200:1.0:1784949328.500125:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000001:2.0:1784949328.500131:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949328.500134:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949328.500137:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:2.0:1784949328.500140:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000400:00000200:2.0:1784949328.500145:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949328.500154:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949328.500160:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949328.500162:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3913:3913:232:4294967295] 192.168.202.137@tcp LPNI seq info [2494:2494:8:4294967295] 00000400:00000200:1.0:1784949328.500168:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949328.500173:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949328.500177:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949328.500181:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000800:00000010:1.0:1784949328.500183:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cf155122. 00000400:00000010:2.0:1784949328.500187:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004520da55. 00000800:00000200:1.0:1784949328.500191:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784949328.500197:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-110 00000800:00000200:1.0:1784949328.500201:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:2.0:1784949328.500205:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -110 00000800:00000200:1.0:1784949328.500207:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000cf155122 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949328.500213:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:1.0:1784949328.500223:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11602 health :900 00000800:00000200:3.0:1784949328.500346:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949328.500349:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000cf155122. 00000400:00000200:3.0:1784949328.500352:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949328.500356:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949328.500359:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949328.500362:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949328.500369:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949328.500377:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949328.500380:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949328.500383:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949328.500387:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949328.500398:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949328.500407:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949328.500414:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949328.500417:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3914:3914:232:4294967295] 192.168.202.137@tcp LPNI seq info [2495:2495:8:4294967295] 00000400:00000200:3.0:1784949328.500423:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949328.500426:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949328.500428:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949328.500432:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949328.500435:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949328.500437:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949328.500465:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949328.500495:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949328.500501:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949328.500502:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949328.500504:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949328.500505:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000800:00000001:0.0:1784949328.500509:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949328.500510:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949328.500518:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949328.500520:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949328.500532:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x96d5 00000800:00000001:0.0:1784949328.500536:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949328.500543:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949328.500544:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949328.500547:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949328.500551:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076145f95 00000400:00000010:0.0:1784949328.500553:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000076145f95. 00000400:00000200:0.0:1784949328.500555:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949328.500558:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000010:0.0:1784949328.500561:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000467d4d37. 00000800:00000001:0.0:1784949328.500566:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949328.500567:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949328.500568:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949329.139967:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949329.139973:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949329.139978:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949329.139979:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949329.140054:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949329.140058:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949329.140060:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949329.332022:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949329.332034:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949329.332036:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949329.332040:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949329.332043:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949329.332046:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949329.332048:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949329.523994:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.5@tcp 00000800:00000010:3.0:1784949329.523999:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004b264730. 00000400:00000200:3.0:1784949329.524006:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949329.524010:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949329.524015:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11602 health :900 00000400:00000200:3.0:1784949329.524019:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949329.524022:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:3.0:1784949329.524025:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000800:00000100:3.0:1784949329.524045:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.10@tcp 00000800:00000010:3.0:1784949329.524046:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002ddfcc2d. 00000400:00000200:3.0:1784949329.524049:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949329.524053:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949329.524055:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949329.524059:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:2.0:1784949329.524062:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949329.524063:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:2.0:1784949329.524073:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2861 00000400:00000001:2.0:1784949329.524080:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949329.524081:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949329.524084:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007d821c01 00000400:00000010:2.0:1784949329.524087:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007d821c01. 00000400:00000200:2.0:1784949329.524091:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949329.524094:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:2.0:1784949329.524097:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000bf0e6e18. 00000400:00000200:2.0:1784949329.524101:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-110 00000400:00000200:2.0:1784949329.524103:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -110 00000400:00000200:2.0:1784949329.524106:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:2.0:1784949329.524108:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2861 00000400:00000001:2.0:1784949329.524110:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949329.524110:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949329.524111:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:2.0:1784949329.524112:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000400:00000200:2.0:1784949329.524114:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949329.524115:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:2.0:1784949329.524116:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000400:00000200:2.0:1784949329.524120:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-110 00000400:00000200:2.0:1784949329.524121:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -110 00000400:00000200:2.0:1784949329.524122:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00010000:00000001:2.0:1784949330.163987:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949330.163993:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949330.163998:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949330.163999:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949330.164073:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949330.164077:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949330.164080:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949330.356072:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949330.356084:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949330.356086:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949330.356090:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949330.356093:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949330.356096:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949330.356098:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949330.548006:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:3.0:1784949330.548006:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.9@tcp 00000800:00000010:3.0:1784949330.548014:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c158c1d0. 00000400:00000010:1.0:1784949330.548015:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000400:00000010:1.0:1784949330.548024:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000467d4d37. 00000400:00000200:3.0:1784949330.548024:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:1.0:1784949330.548032:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000200:3.0:1784949330.548037:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784949330.548042:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000006b55cc75. 00000400:00000200:1.0:1784949330.548051:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949330.548051:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11603 health :700 00000400:00000200:3.0:1784949330.548060:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949330.548066:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:1.0:1784949330.548067:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949330.548076:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949330.548079:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3915:3915:234:4294967295] 192.168.202.137@tcp LPNI seq info [2496:2496:8:4294967295] 00000400:00000200:1.0:1784949330.548083:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949330.548087:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949330.548091:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949330.548095:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000800:00000010:1.0:1784949330.548096:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000800:00000200:1.0:1784949330.548104:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949330.548111:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949330.548115:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000001ee120ca type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949330.548136:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000100:3.0:1784949330.548144:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.14@tcp 00000400:00000200:2.0:1784949330.548146:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2861 00000800:00000010:3.0:1784949330.548150:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000001:2.0:1784949330.548153:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949330.548156:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949330.548159:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:2.0:1784949330.548162:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000400:00000200:2.0:1784949330.548166:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949330.548168:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:2.0:1784949330.548171:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000005cc3cd5d. 00000400:00000200:2.0:1784949330.548175:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:2.0:1784949330.548177:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000400:00000200:2.0:1784949330.548179:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000800:00000200:0.0:1784949330.548237:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949330.548251:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000400:00000200:0.0:1784949330.548255:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949330.548255:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949330.548261:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949330.548265:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:1.0:1784949330.548266:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949330.548270:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784949330.548271:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949330.548277:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949330.548280:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949330.548286:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949330.548291:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949330.548297:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949330.548301:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949330.548303:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:3.0:1784949330.548309:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:1.0:1784949330.548314:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784949330.548325:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949330.548330:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3916:3916:236:4294967295] 192.168.202.137@tcp LPNI seq info [2497:2497:8:4294967295] 00000400:00000200:1.0:1784949330.548338:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784949330.548343:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949330.548345:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c53910d9. 00000400:00000200:2.0:1784949330.548346:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:1.0:1784949330.548350:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:2.0:1784949330.548352:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2861 00000400:00000001:2.0:1784949330.548357:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:1.0:1784949330.548358:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000001:2.0:1784949330.548361:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1784949330.548362:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c53910d9 type 3, nob 144 niov 1 nkiov 1 00000400:00000200:2.0:1784949330.548363:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000af21ddf0 00000400:00000010:2.0:1784949330.548366:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000af21ddf0. 00000400:00000200:2.0:1784949330.548369:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949330.548372:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:2.0:1784949330.548375:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001c9428b0. 00000400:00000200:2.0:1784949330.548378:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-110 00000400:00000200:2.0:1784949330.548380:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -110 00000400:00000200:2.0:1784949330.548382:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00000800:00000001:0.0:1784949330.548400:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949330.548429:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949330.548434:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949330.548436:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949330.548438:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949330.548438:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949330.548442:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c53910d9. 00000400:00000200:0.0:1784949330.548447:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:1.0:1784949330.548448:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949330.548455:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949330.548461:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949330.548463:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x96d9 00000800:00000001:0.0:1784949330.548470:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949330.548480:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949330.548482:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949330.548485:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949330.548490:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc970d6e 00000400:00000010:0.0:1784949330.548491:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bc970d6e. 00000400:00000200:0.0:1784949330.548493:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949330.548496:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000010:0.0:1784949330.548498:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000467d4d37. 00000800:00000001:0.0:1784949330.548501:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949330.548503:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949331.188016:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949331.188022:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949331.188027:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949331.188028:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949331.188083:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949331.188088:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949331.188092:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949331.379973:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949331.379985:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949331.379987:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949331.379990:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949331.379991:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949331.379995:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949331.379999:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949331.380000:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949331.380002:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949331.380012:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000040:3.0:1784949331.380019:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949331.380024:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949331.380028:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949331.380033:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949331.380041:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c9dda688. 00000100:00000001:3.0:1784949331.380049:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949331.380058:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949331.380077:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949331.380080:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949331.380086:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000774d655b] to pc [ptlrpcd_00_03+3] req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.380113:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949331.380116:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949331.380120:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949331.380123:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949331.380124:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949331.380130:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00100000:2.0:1784949331.380131:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:1.0:1784949331.380132:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1784949331.380134:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000100:00000001:1.0:1784949331.380136:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.380137:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1784949331.380140:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:3.0:1784949331.380144:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949331.380146:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784949331.380147:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949331.380149:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949331.380151:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.380154:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949331.380156:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1784949331.380158:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 00000100:00000001:2.0:1784949331.380159:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949331.380163:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949331.380167:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949331.380174:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949331.380187:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949331.380191:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949331.380197:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784949331.380198:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949331.380200:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949331.380202:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949331.380222:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949331.380224:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289689984:0@lo:400: 00000100:00000001:2.0:1784949331.380230:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949331.380236:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1784949331.380239:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e14e991b] to pc [ptlrpcd_00_00+0] req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949331.380240:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949331.380246:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949331.380253:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001c9428b0. 02000000:00000001:2.0:1784949331.380256:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949331.380263:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bcdfbf21. 00000100:00000001:3.0:1784949331.380271:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949331.380272:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000001:1.0:1784949331.380279:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949331.380281:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11608) 00000100:00000001:1.0:1784949331.380282:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:2.0:1784949331.380283:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289689984, portal 25 00000100:00000001:1.0:1784949331.380286:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949331.380289:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949331.380292:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000100:00000040:1.0:1784949331.380293:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.380303:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1784949331.380303:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.380306:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949331.380311:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949331.380312:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949331.380314:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949331.380318:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949331.380320:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.380322:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1784949331.380327:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289690112:192.168.202.137@tcp:400: 00000100:00000040:2.0:1784949331.380328:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784949331.380333:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:2.0:1784949331.380333:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 02000000:00000001:1.0:1784949331.380335:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949331.380338:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949331.380338:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289689984, offset 0 02000000:00000001:1.0:1784949331.380340:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949331.380346:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000467d4d37. 02000000:00000001:1.0:1784949331.380349:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949331.380350:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:1.0:1784949331.380354:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002f6a97f7. 00000400:00000010:1.0:1784949331.380360:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000100:00000200:1.0:1784949331.380363:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289690112, portal 10 00000100:00000001:1.0:1784949331.380367:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:2.0:1784949331.380368:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949331.380369:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000100:00000040:1.0:1784949331.380377:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.380383:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1784949331.380386:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ef80 00000100:00000040:1.0:1784949331.380387:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949331.380392:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000200:2.0:1784949331.380398:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8cd5 [2] + 0 00000400:00000010:2.0:1784949331.380404:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d30ef19f. 00000400:00000200:2.0:1784949331.380408:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000be708ecf 00000100:00000200:1.0:1784949331.380411:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289690112, offset 0 00000400:00000200:2.0:1784949331.380416:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949331.380417:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949331.380424:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949331.380428:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949331.380433:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949331.380437:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949331.380440:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949331.380443:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949331.380446:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3917:3917:236:4294967295] 192.168.202.137@tcp LPNI seq info [2498:2498:8:4294967295] 00000400:00000200:1.0:1784949331.380456:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949331.380461:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949331.380463:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000be708ecf 00000800:00000010:1.0:1784949331.380465:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c53910d9. 00000400:00000010:2.0:1784949331.380468:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000be708ecf. 00000800:00000200:1.0:1784949331.380474:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:2.0:1784949331.380475:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949331.380480:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:1.0:1784949331.380485:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000040:2.0:1784949331.380485:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000974405de x1871651289689984 msgsize 224 00000800:00000200:1.0:1784949331.380491:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c53910d9 type 1, nob 320 niov 1 nkiov 1 00000100:00100000:2.0:1784949331.380497:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:2.0:1784949331.380500:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:2.0:1784949331.380511:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949331.380514:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949331.380519:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2253 usecs (3 jiffies) 00000400:00000200:2.0:1784949331.380520:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000100:00000001:1.0:1784949331.380524:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306270723 00000400:00000010:2.0:1784949331.380524:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:2.0:1784949331.380530:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:1.0:1784949331.380531:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000200:2.0:1784949331.380535:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.380540:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949331.380548:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306270723 00000100:00000001:2.0:1784949331.380551:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000010:1.0:1784949331.380552:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:1.0:1784949331.380556:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000040:2.0:1784949331.380558:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.380560:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306270723 00000100:00000010:1.0:1784949331.380567:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000054c9929b. 00000100:00000001:1.0:1784949331.380571:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949331.380573:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949331.380576:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306270723 00000100:00000001:2.0:1784949331.380577:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:1.0:1784949331.380581:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a6e97b61. 00000100:00000001:1.0:1784949331.380584:0:337119:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949331.380584:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949331.380586:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949331.380590:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306270762 00000100:00000001:1.0:1784949331.380592:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949331.380596:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949331.380597:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949331.380599:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306270723 00000100:00000001:1.0:1784949331.380601:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949331.380604:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949331.380607:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306270762 00000100:00000001:1.0:1784949331.380608:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949331.380610:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949331.380615:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306270723 00000100:00000001:2.0:1784949331.380620:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306270762 00000100:00000001:2.0:1784949331.380625:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306270723 00000100:00000001:2.0:1784949331.380629:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306270762 00000100:00000001:2.0:1784949331.380634:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306270723 00000100:00000001:2.0:1784949331.380639:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949331.380641:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.380644:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.380647:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949331.380653:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949331.380656:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289689984 02000000:00000001:1.0:1784949331.380659:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949331.380660:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949331.380662:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949331.380665:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949331.380668:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289689984 00000020:00000001:1.0:1784949331.380670:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949331.380672:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:1.0:1784949331.380674:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949331.380677:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:1.0:1784949331.380679:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:1.0:1784949331.380682:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949331.380685:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:0.0:1784949331.380687:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:1.0:1784949331.380688:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000800:00000001:2.0:1784949331.380693:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949331.380694:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000010:0.0:1784949331.380695:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c53910d9. 00000020:00000010:1.0:1784949331.380698:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000400:00000200:0.0:1784949331.380703:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949331.380706:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949331.380707:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000013c6c1. 00000400:00000200:0.0:1784949331.380712:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784949331.380712:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:1.0:1784949331.380714:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006df70064. 00000400:00000200:0.0:1784949331.380716:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000800:00000001:2.0:1784949331.380716:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949331.380719:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000020:00000010:1.0:1784949331.380719:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949331.380724:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1784949331.380725:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:1.0:1784949331.380730:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000200:0.0:1784949331.380731:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.380735:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:2.0:1784949331.380735:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f000 00000100:00000001:1.0:1784949331.380738:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949331.380742:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949331.380746:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949331.380746:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 181360 00000100:00000040:0.0:1784949331.380751:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.380752:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949331.380753:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949331.380761:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949331.380762:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949331.380764:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1784949331.380764:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949331.380766:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949331.380771:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949331.380788:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:1.0:1784949331.380790:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000200:2.0:1784949331.380791:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:1.0:1784949331.380792:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949331.380798:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 829 00000100:00000010:2.0:1784949331.380799:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d482a903. 00000100:00000040:1.0:1784949331.380801:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949331.380805:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180543568 : -115539529008048 : ffff96eadb6a7c50) 00000100:00000040:2.0:1784949331.380806:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000d482a903 x1871651289690112 msgsize 224 00000100:00000001:0.0:1784949331.380807:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949331.380810:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000974405de x1871651289689984/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:237/0 lens 224/0 e 0 to 0 dl 1784949342 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949331.380812:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949331.380820:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949331.380826:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949331.380829:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000010:1.0:1784949331.380830:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000e424fdc3. 00000100:00000001:1.0:1784949331.380835:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949331.380837:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000e424fdc3. 00000800:00000001:2.0:1784949331.380838:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:1.0:1784949331.380842:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000974405de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289689984:12345-0@lo:400: 00000800:00000001:2.0:1784949331.380844:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949331.380846:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289689984 00000020:00000001:1.0:1784949331.380850:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:3.0:1784949331.380852:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000001:1.0:1784949331.380854:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949331.380856:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949331.380856:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289690112 00000020:00000001:1.0:1784949331.380860:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:3.0:1784949331.380862:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:1.0:1784949331.380863:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949331.380867:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:1.0:1784949331.380868:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949331.380871:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949331.380872:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949331.380874:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000001:3.0:1784949331.380877:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949331.380878:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949331.380881:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949331.380883:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289690112 00000020:00000001:1.0:1784949331.380884:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949331.380887:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:3.0:1784949331.380888:0:342762:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1784949331.380890:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000040:3.0:1784949331.380892:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 02000000:00000010:1.0:1784949331.380895:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000049c99d00. 00000020:00000001:3.0:1784949331.380896:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1784949331.380899:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949331.380901:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:1.0:1784949331.380902:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949331.380925:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949331.380927:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289689984 00010000:00000001:1.0:1784949331.380930:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949331.380934:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000974405de x1871651289689984/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:237/0 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949331.380935:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949331.380943:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:1.0:1784949331.380945:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949331.380948:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949331.380949:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00001000:1.0:1784949331.380951:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=53 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949331.380955:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1784949331.380955:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000600bc50e. 00000100:00000040:1.0:1784949331.380959:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949331.380962:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949331.380963:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:1.0:1784949331.380967:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000010:3.0:1784949331.380969:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000600bc50e. 02000000:00000001:1.0:1784949331.380970:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949331.380975:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949331.380978:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:3.0:1784949331.380979:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000052e1e01a. 00000400:00000010:1.0:1784949331.380980:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000100:00000200:1.0:1784949331.380985:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289689984, offset 224 00000020:00000010:3.0:1784949331.380987:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ce2d3338. 00000400:00000200:1.0:1784949331.380991:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:3.0:1784949331.380993:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000200:1.0:1784949331.380998:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949331.381004:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:1.0:1784949331.381007:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ef80 00000100:00000001:3.0:1784949331.381010:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000400:00000200:1.0:1784949331.381013:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x96dd [1] + 224 00000100:00000001:3.0:1784949331.381014:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949331.381019:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949331.381020:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949331.381026:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949331.381028:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949331.381030:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949331.381033:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:1.0:1784949331.381035:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381037:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949331.381048:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381056:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949331.381057:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784949331.381062:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000400:00000010:1.0:1784949331.381065:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000100:00000001:1.0:1784949331.381068:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949331.381069:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:1.0:1784949331.381073:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949331.381074:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:1.0:1784949331.381077:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:2.0:1784949331.381077:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949331.381078:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949331.381082:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1784949331.381083:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949331.381086:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949331.381090:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00100000:3.0:1784949331.381090:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 805 00000100:00000001:0.0:1784949331.381091:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000001:2.0:1784949331.381091:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1784949331.381094:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000974405de x1871651289689984/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:237/0 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949331.381095:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000001:2.0:1784949331.381096:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1784949331.381099:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949331.381100:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949331.381100:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bcdfbf21. 00000100:00000001:3.0:1784949331.381106:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469574656 : -115544239976960 : ffff96e9c29ece00) 00000100:00000001:0.0:1784949331.381107:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949331.381109:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000100:00100000:1.0:1784949331.381112:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000974405de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289689984:12345-0@lo:400: Request processed in 281us (623us total) trans 0 rc 0/0 00000400:00000010:2.0:1784949331.381114:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00000040:3.0:1784949331.381119:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000d482a903 x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:237/0 lens 224/0 e 0 to 0 dl 1784949342 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949331.381122:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784949331.381123:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 829 00000100:00000040:1.0:1784949331.381128:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000200:2.0:1784949331.381130:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949331.381133:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949331.381136:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949331.381141:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006df70064. 00000020:00000010:1.0:1784949331.381146:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000100:00000001:3.0:1784949331.381149:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949331.381152:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000013c6c1. 00000020:00000010:3.0:1784949331.381155:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000600bc50e. 00000020:00000040:1.0:1784949331.381158:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:2.0:1784949331.381159:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381160:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949331.381163:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:1.0:1784949331.381166:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000049c99d00. 00000020:00000010:3.0:1784949331.381167:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000600bc50e. 02000000:00000001:1.0:1784949331.381172:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949331.381176:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949331.381179:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1784949331.381181:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000d482a903 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289690112:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949331.381182:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:2.0:1784949331.381183:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949331.381187:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000aba79327. 00000100:00000001:2.0:1784949331.381188:0:337121:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:3.0:1784949331.381190:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289690112 02000000:00000001:2.0:1784949331.381193:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000400:00000010:1.0:1784949331.381194:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000298e1506. 00000020:00000001:3.0:1784949331.381195:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949331.381197:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949331.381200:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949331.381201:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:2.0:1784949331.381202:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949331.381216:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949331.381218:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=53 v=5 (5 5 5 5) 00000020:00000001:3.0:1784949331.381221:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00001000:2.0:1784949331.381223:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=53 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949331.381226:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784949331.381227:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949331.381229:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949331.381233:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949331.381233:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00010000:00000001:2.0:1784949331.381237:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949331.381239:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949331.381240:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949331.381246:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:2.0:1784949331.381247:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949331.381251:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949331.381255:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.381256:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949331.381261:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949331.381262:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949331.381262:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:2.0:1784949331.381267:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381268:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949331.381272:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:2.0:1784949331.381276:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000774d655b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289689984:0@lo:400: 02000000:00000010:3.0:1784949331.381279:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d78419ed. 00000100:00000001:2.0:1784949331.381280:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949331.381285:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1784949331.381285:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949331.381288:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000774d655b x1871651289689984/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381291:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.381296:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1784949331.381297:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949331.381298:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949331.381302:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001c9428b0. 00010000:00000040:3.0:1784949331.381303:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289690112 02000000:00000001:2.0:1784949331.381306:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00010000:00000001:3.0:1784949331.381308:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:2.0:1784949331.381309:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949331.381312:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949331.381314:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949331.381317:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c9dda688. 00010000:00000200:3.0:1784949331.381317:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000d482a903 x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:237/0 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949331.381320:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949331.381323:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949331.381325:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000774d655b. 00000100:00000001:2.0:1784949331.381329:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949331.381330:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949331.381334:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949331.381337:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:2.0:1784949331.381338:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949331.381341:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949331.381342:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949331.381343:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949331.381345:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949331.381348:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=48 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949331.381354:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949331.381357:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949331.381360:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949331.381362:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949331.381365:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949331.381368:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949331.381370:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949331.381375:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000014d7b07a. 00000100:00000200:3.0:1784949331.381379:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289690112, offset 224 00000400:00000200:3.0:1784949331.381384:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949331.381392:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949331.381397:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3918:3918:236:4294967295] 192.168.202.137@tcp LPNI seq info [2499:2499:8:4294967295] 00000400:00000200:3.0:1784949331.381406:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949331.381411:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949331.381415:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949331.381419:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949331.381425:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949331.381428:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949331.381443:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949331.381446:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949331.381449:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949331.381450:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949331.381452:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949331.381457:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d482a903 x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:237/0 lens 224/224 e 0 to 0 dl 1784949342 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949331.381468:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000d482a903 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289690112:12345-192.168.202.137@tcp:400: Request processed in 318us (660us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949331.381476:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 805 00000100:00000040:3.0:1784949331.381479:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949331.381482:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949331.381484:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949331.381487:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ce2d3338. 00000020:00000010:3.0:1784949331.381490:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949331.381495:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000052e1e01a. 00000020:00000040:3.0:1784949331.381501:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949331.381504:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949331.381524:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949331.381527:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000400:00000200:0.0:1784949331.381530:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949331.381533:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949331.381535:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000014d7b07a 00000400:00000010:0.0:1784949331.381537:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000014d7b07a. 00000100:00000001:0.0:1784949331.381539:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949331.381540:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949331.381544:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949331.381547:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949331.381548:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949331.381549:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949331.381552:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949331.381559:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f000 00000400:00000200:0.0:1784949331.381563:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x96e5 [1] + 224 00000800:00000001:0.0:1784949331.381566:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949331.381570:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949331.381572:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949331.381575:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949331.381578:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949331.381580:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949331.381585:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949331.381596:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949331.381599:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949331.381601:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949331.381634:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949331.381638:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949331.381642:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949331.381643:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949331.381645:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002f6a97f7. 00000400:00000200:3.0:1784949331.381650:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:3.0:1784949331.381651:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000100:00000001:3.0:1784949331.381654:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949331.381659:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949331.381669:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381677:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949331.381678:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949331.381680:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949331.381682:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949331.381683:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949331.381685:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949331.381687:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949331.381692:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=48 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949331.381696:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=48 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949331.381698:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949331.381700:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949331.381702:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949331.381705:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949331.381713:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949331.381716:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949331.381718:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949331.381720:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949331.381721:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949331.381725:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381739:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949331.381742:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949331.381743:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949331.381746:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949331.381755:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000e14e991b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289690112:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949331.381759:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949331.381761:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949331.381764:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000e14e991b x1871651289690112/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949331.381771:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949331.381772:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949331.381774:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000467d4d37. 02000000:00000001:3.0:1784949331.381778:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949331.381779:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949331.381781:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949331.381783:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949331.381784:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:3.0:1784949331.381787:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949331.381788:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949331.381789:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000e14e991b. 00000100:00000001:3.0:1784949331.381802:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949331.381803:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949331.381805:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949331.381809:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949331.381810:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949331.381812:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949331.381814:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949331.422973:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306270762, at 4306270763 00000100:00000040:0.0:1784949331.422976:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306270762, at 4306270763 00000100:00000040:1.0:1784949331.422976:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306270762, at 4306270763 00000100:00000010:3.0:1784949331.422980:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a6e97b61. 00000100:00000040:2.0:1784949331.422981:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306270762, at 4306270763 00000100:00000010:1.0:1784949331.422985:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:0.0:1784949331.422988:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000010:2.0:1784949331.422989:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000054c9929b. 00010000:00000001:2.0:1784949332.211993:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949332.212000:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949332.212005:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949332.212007:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949332.212010:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949332.212012:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949332.212015:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949332.212016:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949332.212018:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949332.212019:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949332.212098:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949332.212102:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949332.212104:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949332.404010:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949332.404022:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949332.404024:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949332.404029:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949332.404033:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949332.404036:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949332.404039:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949333.235991:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949333.235998:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949333.236003:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949333.236004:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949333.236087:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949333.236091:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949333.236094:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949333.428009:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949333.428023:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949333.428025:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949333.428030:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949333.428033:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949333.428037:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949333.428039:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949334.260025:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949334.260032:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949334.260052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949334.260054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949334.260162:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949334.260167:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949334.260170:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949334.451977:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949334.451989:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949334.451991:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949334.451995:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949334.451997:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949334.452000:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949334.452002:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949335.284028:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949335.284036:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949335.284044:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949335.284045:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949335.284141:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949335.284147:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949335.284152:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949335.476043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949335.476059:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949335.476063:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949335.476069:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949335.476073:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949335.476077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949335.476080:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949335.887511:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949335.887541:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949335.887551:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949335.887554:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949336.308024:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949336.308032:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949336.308040:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949336.308041:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949336.308237:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949336.308242:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949336.308246:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949336.500019:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949336.500030:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949336.500039:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b83ea808. 00000020:00000040:0.0:1784949336.500047:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949336.500053:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949336.500057:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949336.500062:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949336.500070:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003ed79484. 00000100:00000001:1.0:1784949336.500076:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949336.500078:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500089:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949336.500090:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949336.500093:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949336.500101:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949336.500105:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949336.500108:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949336.500110:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949336.500111:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500116:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949336.500121:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b83ea808] to pc [ptlrpcd_00_01+1] req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949336.500143:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949336.500146:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949336.500150:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949336.500152:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949336.500156:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949336.500159:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784949336.500160:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bab47beb. 00000100:00000001:1.0:1784949336.500180:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1784949336.500182:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949336.500184:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949336.500187:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949336.500190:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1784949336.500193:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949336.500194:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949336.500195:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949336.500201:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949336.500203:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000062b4a223. 00000100:00000001:0.0:1784949336.500211:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500222:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949336.500226:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949336.500228:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1784949336.500236:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949336.500243:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949336.500247:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949336.500249:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949336.500252:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949336.500258:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289690240:0@lo:400: 00000100:00000040:0.0:1784949336.500263:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bab47beb] to pc [ptlrpcd_00_02+2] req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949336.500270:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949336.500280:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949336.500285:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949336.500290:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949336.500297:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000013c6c1. 00000100:00000001:0.0:1784949336.500298:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949336.500301:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949336.500305:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11613) 00000400:00000010:1.0:1784949336.500313:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000035db9be3. 00000100:00000001:3.0:1784949336.500318:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949336.500323:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949336.500324:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1784949336.500326:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000100:00000001:3.0:1784949336.500326:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949336.500329:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949336.500337:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949336.500340:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289690240, portal 25 00000100:00000001:1.0:1784949336.500349:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949336.500354:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949336.500355:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469561856 : -115544239989760 : ffff96e9c29e9c00) 00000100:00000001:3.0:1784949336.500357:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949336.500364:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949336.500366:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949336.500368:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949336.500370:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949336.500380:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289690368:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949336.500392:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949336.500393:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949336.500397:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949336.500402:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949336.500403:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:3.0:1784949336.500407:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:1.0:1784949336.500411:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 02000000:00000010:3.0:1784949336.500414:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000bf393d23. 00000100:00000200:1.0:1784949336.500417:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289690240, offset 0 02000000:00000001:3.0:1784949336.500418:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949336.500429:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000400:00000200:1.0:1784949336.500435:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784949336.500446:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00000200:3.0:1784949336.500453:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289690368, portal 10 00000100:00000001:3.0:1784949336.500457:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949336.500459:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949336.500460:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469556480 : -115544239995136 : ffff96e9c29e8700) 00000100:00000040:3.0:1784949336.500471:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949336.500483:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f080 00000100:00000001:3.0:1784949336.500483:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949336.500490:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949336.500495:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8cf5 [2] + 0 00000400:00000010:3.0:1784949336.500495:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004df903a5. 00000400:00000010:1.0:1784949336.500504:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a31fc1e7. 00000400:00000200:1.0:1784949336.500509:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000fea29ebd 00000100:00000200:3.0:1784949336.500513:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289690368, offset 0 00000400:00000200:1.0:1784949336.500519:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949336.500525:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949336.500530:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949336.500540:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949336.500546:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949336.500550:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949336.500554:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949336.500557:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949336.500562:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3919:3919:236:4294967295] 192.168.202.137@tcp LPNI seq info [2500:2500:8:4294967295] 00000400:00000200:3.0:1784949336.500572:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949336.500579:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949336.500580:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fea29ebd 00000400:00000010:1.0:1784949336.500585:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000fea29ebd. 00000800:00000010:3.0:1784949336.500587:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000100:00000001:1.0:1784949336.500591:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949336.500595:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:3.0:1784949336.500601:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:1.0:1784949336.500602:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000475c8050 x1871651289690240 msgsize 224 00000100:00100000:1.0:1784949336.500613:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949336.500619:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:3.0:1784949336.500619:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949336.500625:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949336.500637:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949336.500644:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949336.500647:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000400:00000010:1.0:1784949336.500650:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000100:00000040:3.0:1784949336.500652:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2253 usecs (3 jiffies) 00000100:00000001:1.0:1784949336.500654:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1784949336.500659:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306275843 00000100:00000200:1.0:1784949336.500662:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784949336.500666:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00000001:3.0:1784949336.500677:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949336.500678:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949336.500685:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949336.500690:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949336.500692:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306275843 00000100:00000001:1.0:1784949336.500693:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949336.500699:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289690240 00000100:00000001:1.0:1784949336.500701:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:3.0:1784949336.500701:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 02000000:00000001:2.0:1784949336.500706:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949336.500709:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949336.500711:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949336.500711:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949336.500714:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306275843 00000100:00000001:2.0:1784949336.500717:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500719:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306275882 00000100:00000010:3.0:1784949336.500720:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 02000000:00000001:2.0:1784949336.500725:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500730:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306275843 00000100:00000001:3.0:1784949336.500730:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00100000:2.0:1784949336.500731:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289690240 00000020:00000001:2.0:1784949336.500738:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949336.500741:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949336.500743:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306275882 00000100:00000001:3.0:1784949336.500744:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306275843 00000020:00000001:2.0:1784949336.500745:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949336.500751:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:1.0:1784949336.500752:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306275843 00000100:00000010:3.0:1784949336.500753:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000020:00000001:2.0:1784949336.500757:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949336.500761:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306275882 00000020:00000001:2.0:1784949336.500763:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949336.500764:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949336.500766:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306275843 00000100:00000001:2.0:1784949336.500770:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949336.500776:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949336.500777:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306275882 00000020:00000010:2.0:1784949336.500777:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b4894542. 00000100:00000001:2.0:1784949336.500785:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949336.500787:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500788:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306275843 00000020:00000010:2.0:1784949336.500789:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b4894542. 00000100:00000001:3.0:1784949336.500799:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500800:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949336.500802:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001c9428b0. 00000100:00000001:3.0:1784949336.500807:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949336.500808:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949336.500811:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000080a10e75. 00000100:00000001:1.0:1784949336.500815:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949336.500816:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949336.500819:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784949336.500819:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000100:00000001:3.0:1784949336.500821:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1784949336.500829:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949336.500830:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949336.500831:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949336.500835:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:2.0:1784949336.500835:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949336.500837:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949336.500842:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949336.500845:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949336.500845:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949336.500846:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000200:0.0:1784949336.500848:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949336.500848:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949336.500853:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949336.500855:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000100:00000001:1.0:1784949336.500855:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.500862:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949336.500864:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949336.500864:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949336.500875:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949336.500880:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004df903a5 00000100:00000001:2.0:1784949336.500881:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949336.500882:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004df903a5. 00000100:00000001:0.0:1784949336.500890:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949336.500896:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949336.500900:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949336.500902:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949336.500940:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949336.500944:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949336.500944:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949336.500951:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 830 00000800:00000001:1.0:1784949336.500952:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949336.500954:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949336.500959:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000800:00000001:1.0:1784949336.500960:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949336.500966:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180529232 : -115539529022384 : ffff96eadb6a4450) 00000100:00000001:0.0:1784949336.500970:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949336.500975:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1784949336.500977:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000475c8050 x1871651289690240/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:242/0 lens 224/0 e 0 to 0 dl 1784949347 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949336.500978:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949336.500993:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f100 00000100:00000001:2.0:1784949336.500994:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784949336.501001:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 181584 00000020:00000010:2.0:1784949336.501001:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b4894542. 00000100:00000001:2.0:1784949336.501007:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000001:1.0:1784949336.501011:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000010:2.0:1784949336.501012:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b4894542. 00000100:00100000:2.0:1784949336.501021:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000475c8050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289690240:12345-0@lo:400: 00000800:00000001:1.0:1784949336.501022:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949336.501027:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1784949336.501030:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289690240 00000400:00000200:1.0:1784949336.501036:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949336.501036:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949336.501042:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949336.501044:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:2.0:1784949336.501046:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949336.501047:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:2.0:1784949336.501052:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000010:1.0:1784949336.501054:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000008223e251. 00000020:00000001:2.0:1784949336.501057:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000040:1.0:1784949336.501059:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000008223e251 x1871651289690368 msgsize 224 00000020:00000001:2.0:1784949336.501062:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949336.501067:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949336.501071:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:2.0:1784949336.501072:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949336.501077:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949336.501079:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949336.501081:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949336.501083:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949336.501087:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949336.501088:0:337113:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:2.0:1784949336.501092:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000007672855d. 00000800:00000001:1.0:1784949336.501097:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:2.0:1784949336.501097:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949336.501099:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949336.501103:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949336.501103:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949336.501107:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289690240 00010000:00000001:2.0:1784949336.501111:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949336.501116:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000475c8050 x1871651289690240/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:242/0 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949336.501124:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949336.501126:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949336.501131:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00001000:2.0:1784949336.501132:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=58 v=5 (1 1 1 1) 00000100:00100000:3.0:1784949336.501136:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289690368 00000100:00000001:2.0:1784949336.501138:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784949336.501140:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949336.501143:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:2.0:1784949336.501145:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949336.501146:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949336.501150:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:3.0:1784949336.501152:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949336.501154:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00100000:3.0:1784949336.501157:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289690368 02000000:00000001:2.0:1784949336.501158:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949336.501176:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949336.501179:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:2.0:1784949336.501180:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1784949336.501185:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:2.0:1784949336.501190:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000040:3.0:1784949336.501191:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000010:2.0:1784949336.501195:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000020:00000001:3.0:1784949336.501196:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949336.501201:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:2.0:1784949336.501204:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289690240, offset 224 00000100:00000001:3.0:1784949336.501206:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784949336.501212:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000010:3.0:1784949336.501212:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000600bc50e. 00000100:00000001:3.0:1784949336.501219:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949336.501222:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000600bc50e. 00000400:00000200:2.0:1784949336.501227:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000010:3.0:1784949336.501229:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f047fe3. 00000020:00000010:3.0:1784949336.501235:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003f7291f9. 00000400:00000200:2.0:1784949336.501239:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f080 00000020:00000010:3.0:1784949336.501243:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000200:2.0:1784949336.501249:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x96f9 [1] + 224 00000100:00000040:3.0:1784949336.501253:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:2.0:1784949336.501256:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949336.501258:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949336.501261:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:2.0:1784949336.501263:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949336.501266:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949336.501269:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949336.501273:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:2.0:1784949336.501277:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949336.501278:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949336.501282:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949336.501294:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949336.501299:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949336.501313:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949336.501317:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949336.501321:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949336.501324:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:2.0:1784949336.501330:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000100:00100000:3.0:1784949336.501331:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 806 00000100:00000001:2.0:1784949336.501338:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:3.0:1784949336.501339:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949336.501343:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949336.501344:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637810048 : -115544071741568 : ffff96e9cca5df80) 00000100:00000001:1.0:1784949336.501346:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949336.501350:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.501351:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784949336.501353:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000008223e251 x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:242/0 lens 224/0 e 0 to 0 dl 1784949347 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949336.501359:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784949336.501360:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000400:00000001:1.0:1784949336.501365:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00010000:00000001:2.0:1784949336.501368:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000010:1.0:1784949336.501370:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000035db9be3. 00000020:00000001:2.0:1784949336.501374:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949336.501379:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:1.0:1784949336.501380:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000020:00000001:2.0:1784949336.501380:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000010:3.0:1784949336.501383:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000600bc50e. 00000400:00000010:1.0:1784949336.501385:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 00000100:00000040:2.0:1784949336.501389:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000475c8050 x1871651289690240/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:242/0 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949336.501390:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949336.501392:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949336.501398:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000600bc50e. 00000100:00000200:1.0:1784949336.501401:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949336.501412:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000008223e251 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289690368:12345-192.168.202.137@tcp:400: 00000100:00100000:2.0:1784949336.501422:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000475c8050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289690240:12345-0@lo:400: Request processed in 422us (814us total) trans 0 rc 0/0 00000100:00000200:3.0:1784949336.501428:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289690368 00000020:00000001:3.0:1784949336.501433:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949336.501439:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949336.501443:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949336.501444:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949336.501445:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 830 00000020:00000001:3.0:1784949336.501448:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:2.0:1784949336.501455:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:3.0:1784949336.501455:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:1.0:1784949336.501460:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949336.501462:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:3.0:1784949336.501463:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949336.501465:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949336.501466:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:3.0:1784949336.501468:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949336.501470:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000001:3.0:1784949336.501472:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000010:2.0:1784949336.501474:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000080a10e75. 02000000:00000001:1.0:1784949336.501475:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000001:3.0:1784949336.501476:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:1.0:1784949336.501481:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:3.0:1784949336.501482:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949336.501483:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000100:00000001:1.0:1784949336.501485:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949336.501487:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000010:2.0:1784949336.501491:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001c9428b0. 02000000:00000001:1.0:1784949336.501492:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949336.501493:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000040:2.0:1784949336.501498:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:3.0:1784949336.501499:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00001000:1.0:1784949336.501501:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=58 v=5 (5 5 5 5) 02000000:00000010:3.0:1784949336.501504:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000058a72386. 02000000:00000001:2.0:1784949336.501506:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00001000:1.0:1784949336.501507:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=58 v=5 (1 1 1 1) 02000000:00000001:3.0:1784949336.501510:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949336.501511:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000007672855d. 00000100:00000001:1.0:1784949336.501515:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949336.501517:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.501519:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949336.501519:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:3.0:1784949336.501523:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949336.501525:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949336.501526:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000040:3.0:1784949336.501528:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289690368 00010000:00000001:1.0:1784949336.501530:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949336.501533:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:3.0:1784949336.501534:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949336.501535:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949336.501542:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bcdfbf21. 00010000:00000200:3.0:1784949336.501542:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000008223e251 x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:242/0 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949336.501544:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949336.501552:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000341aff9b. 00010000:00000001:3.0:1784949336.501580:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949336.501581:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1784949336.501585:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.501587:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00001000:3.0:1784949336.501589:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=53 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949336.501591:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949336.501600:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949336.501601:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949336.501606:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949336.501613:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949336.501618:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949336.501622:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949336.501626:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289690240:0@lo:400: 00000100:00000001:3.0:1784949336.501628:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949336.501634:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949336.501635:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949336.501641:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1784949336.501642:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000078ee4c1c. 00000100:00000040:1.0:1784949336.501646:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b83ea808 x1871651289690240/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949336.501649:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289690368, offset 224 00000400:00000200:3.0:1784949336.501661:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949336.501662:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949336.501665:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949336.501667:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000013c6c1. 00000400:00000200:3.0:1784949336.501672:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 02000000:00000001:1.0:1784949336.501674:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949336.501679:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949336.501683:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:3.0:1784949336.501683:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3920:3920:236:4294967295] 192.168.202.137@tcp LPNI seq info [2501:2501:8:4294967295] 00000020:00000001:1.0:1784949336.501688:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949336.501692:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003ed79484. 02000000:00000001:1.0:1784949336.501698:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949336.501701:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949336.501705:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b83ea808. 00000400:00000200:3.0:1784949336.501706:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949336.501712:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949336.501716:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949336.501717:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949336.501719:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949336.501724:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000100:00000001:1.0:1784949336.501725:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949336.501728:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949336.501732:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949336.501734:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949336.501740:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949336.501749:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949336.501753:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949336.501770:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949336.501774:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949336.501778:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949336.501780:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949336.501783:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949336.501789:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000008223e251 x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:242/0 lens 224/224 e 0 to 0 dl 1784949347 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949336.501803:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000008223e251 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289690368:12345-192.168.202.137@tcp:400: Request processed in 423us (742us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949336.501813:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 806 00000100:00000040:3.0:1784949336.501816:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949336.501819:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949336.501820:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949336.501823:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003f7291f9. 00000020:00000010:3.0:1784949336.501827:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949336.501830:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f047fe3. 00000020:00000040:3.0:1784949336.501834:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949336.501836:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949336.501874:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949336.501878:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:2.0:1784949336.501881:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949336.501886:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949336.501889:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000078ee4c1c 00000400:00000010:2.0:1784949336.501890:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000078ee4c1c. 00000100:00000001:2.0:1784949336.501893:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949336.501895:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949336.501899:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949336.501928:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949336.501930:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949336.501933:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949336.501939:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949336.501946:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f100 00000400:00000200:2.0:1784949336.501951:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9701 [1] + 224 00000800:00000001:2.0:1784949336.501955:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949336.501959:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949336.501961:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949336.501964:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949336.501967:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949336.501970:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949336.501977:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949336.501991:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949336.501995:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949336.501997:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949336.502024:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949336.502028:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949336.502031:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949336.502033:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949336.502034:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000400:00000200:3.0:1784949336.502039:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:3.0:1784949336.502041:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:3.0:1784949336.502043:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949336.502048:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949336.502056:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949336.502081:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949336.502083:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949336.502085:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949336.502087:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949336.502089:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949336.502091:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949336.502093:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949336.502099:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=53 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949336.502103:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=53 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949336.502106:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949336.502107:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949336.502110:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949336.502115:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949336.502124:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949336.502144:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949336.502145:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949336.502147:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949336.502148:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949336.502150:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949336.502155:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949336.502157:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949336.502158:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949336.502160:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949336.502187:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289690368:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949336.502191:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949336.502193:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949336.502195:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000bab47beb x1871651289690368/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949336.502201:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949336.502202:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949336.502204:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000bf393d23. 02000000:00000001:3.0:1784949336.502206:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949336.502208:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949336.502209:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949336.502210:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949336.502212:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000062b4a223. 02000000:00000001:3.0:1784949336.502214:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949336.502215:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949336.502216:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000bab47beb. 00000100:00000001:3.0:1784949336.502220:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949336.502222:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949336.502224:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949336.502227:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949336.502228:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949336.502229:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949336.502231:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949336.542969:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306275882, at 4306275883 00000100:00000010:0.0:1784949336.542984:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000040:3.0:1784949336.542986:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306275882, at 4306275883 00000100:00000040:2.0:1784949336.542989:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306275882, at 4306275883 00000100:00000040:1.0:1784949336.542992:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306275882, at 4306275883 00000100:00000010:3.0:1784949336.542995:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000010:2.0:1784949336.542997:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:1.0:1784949336.543006:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00010000:00000001:2.0:1784949337.332040:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949337.332049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949337.332056:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949337.332058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949337.332220:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949337.332226:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949337.332231:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949337.524016:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949337.524031:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949337.524034:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949337.524039:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949337.524043:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949337.524047:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949337.524050:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949338.356034:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949338.356044:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949338.356053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949338.356056:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949338.356207:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949338.356212:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949338.356216:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949338.548009:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949338.548026:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949338.548028:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949338.548034:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949338.548038:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949338.548042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949338.548045:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949339.380038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949339.380058:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949339.380066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949339.380068:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949339.380227:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949339.380234:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949339.380239:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949339.572106:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949339.572123:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949339.572126:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949339.572134:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949339.572181:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949339.572186:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949339.572190:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949340.404056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949340.404065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949340.404073:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949340.404075:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949340.404201:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949340.404209:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949340.404215:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949340.596098:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949340.596114:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949340.596116:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949340.596123:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949340.596152:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949340.596156:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949340.596160:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1784949341.108171:0:344053:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.29:988 00000400:00000100:2.0:1784949341.108174:0:344056:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.6:988 00000400:00000100:3.0:1784949341.108203:0:344052:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.14:988 00000400:00000100:1.0:1784949341.108220:0:344054:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.12:988 00000400:00000100:0.0:1784949341.108293:0:344053:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.29@tcp at host 111.222.142.29:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784949341.108293:0:344056:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.6@tcp at host 111.222.142.6:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:3.0:1784949341.108308:0:344052:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.14@tcp at host 111.222.142.14:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784949341.108313:0:344054:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.12@tcp at host 111.222.142.12:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949341.108319:0:344056:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.6@tcp: down 00000400:00000200:0.0:1784949341.108320:0:344053:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.29@tcp: down 00000400:00000200:2.0:1784949341.108336:0:344056:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784949341.108337:0:344052:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.14@tcp: down 00000400:00000200:1.0:1784949341.108340:0:344054:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.12@tcp: down 00000800:00000200:0.0:1784949341.108353:0:344053:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.142.29@tcp down 00000400:00000200:3.0:1784949341.108355:0:344052:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949341.108367:0:344054:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784949341.108379:0:344054:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.12@tcp 00000800:00000010:1.0:1784949341.108385:0:344054:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000aba78344. 00000400:00000200:1.0:1784949341.108396:0:344054:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949341.108406:0:344054:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949341.108417:0:344054:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11614 health :900 00000400:00000200:1.0:1784949341.108425:0:344054:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949341.108429:0:344054:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:1.0:1784949341.108438:0:344054:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000100:3.0:1784949341.108517:0:343954:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.8:988 00000400:00000100:0.0:1784949341.108549:0:344008:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.2:988 00000400:00000200:1.0:1784949341.108561:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949341.108574:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:1.0:1784949341.108584:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949341.108588:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000100:3.0:1784949341.108590:0:343954:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.8@tcp at host 111.222.142.8:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949341.108594:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056dea15c 00000400:00000100:2.0:1784949341.108599:0:343958:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.10:988 00000400:00000010:1.0:1784949341.108604:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000056dea15c. 00000400:00000100:0.0:1784949341.108614:0:344008:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.2@tcp at host 111.222.142.2:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949341.108617:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949341.108618:0:343954:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.8@tcp: down 00000400:00000200:1.0:1784949341.108633:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:0.0:1784949341.108642:0:344008:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.2@tcp: down 00000400:00000100:2.0:1784949341.108642:0:343958:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.10@tcp at host 111.222.142.10:988 took too long: that node may be hung or experiencing high load. 00000400:00000010:1.0:1784949341.108644:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004041a48e. 00000400:00000200:2.0:1784949341.108657:0:343958:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.10@tcp: down 00000400:00000200:3.0:1784949341.108661:0:343954:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949341.108662:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-110 00000400:00000200:0.0:1784949341.108674:0:344008:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949341.108678:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -110 00000800:00000100:3.0:1784949341.108686:0:343954:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.8@tcp 00000400:00000200:2.0:1784949341.108687:0:343958:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1784949341.108694:0:344008:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.2@tcp 00000800:00000010:3.0:1784949341.108700:0:343954:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000eddb6110. 00000400:00000200:1.0:1784949341.108701:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000800:00000010:0.0:1784949341.108705:0:344008:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ffbbc320. 00000400:00000200:0.0:1784949341.108721:0:344008:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949341.108732:0:344008:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949341.108742:0:344008:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000100:2.0:1784949341.108743:0:344011:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.142.4:988 00000400:00000200:0.0:1784949341.108747:0:344008:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:0.0:1784949341.108754:0:344008:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000100:2.0:1784949341.108770:0:344011:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.142.4@tcp at host 111.222.142.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949341.108779:0:344011:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.142.4@tcp: down 00000400:00000200:2.0:1784949341.108783:0:344011:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1784949341.108790:0:344011:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.4@tcp 00000800:00000010:2.0:1784949341.108795:0:344011:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000081367938. 00000400:00000200:2.0:1784949341.108804:0:344011:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949341.108815:0:344011:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949341.108817:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1784949341.108825:0:344011:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949341.108829:0:343954:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949341.108830:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000001:1.0:1784949341.108841:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1784949341.108842:0:344011:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000001:1.0:1784949341.108846:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949341.108854:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000309fc37c 00000400:00000200:3.0:1784949341.108856:0:343954:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949341.108857:0:344011:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:1.0:1784949341.108860:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000309fc37c. 00000400:00000200:1.0:1784949341.108876:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949341.108880:0:343954:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949341.108890:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:3.0:1784949341.108897:0:343954:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000010:1.0:1784949341.108901:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000047e19e91. 00000400:00000200:1.0:1784949341.108961:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-110 00000400:00000200:1.0:1784949341.108965:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -110 00000400:00000200:3.0:1784949341.108982:0:343954:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:1.0:1784949341.108996:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:1.0:1784949341.109022:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:1.0:1784949341.109028:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949341.109031:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949341.109033:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000864ba3de 00000400:00000010:1.0:1784949341.109035:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000864ba3de. 00000400:00000200:1.0:1784949341.109040:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949341.109045:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:1.0:1784949341.109048:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001df3b40e. 00000400:00000200:1.0:1784949341.109052:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-110 00000400:00000200:1.0:1784949341.109055:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -110 00000400:00000200:1.0:1784949341.109059:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:1.0:1784949341.109062:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:1.0:1784949341.109065:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949341.109066:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949341.109068:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006f3c62ee 00000400:00000010:1.0:1784949341.109070:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006f3c62ee. 00000400:00000200:1.0:1784949341.109072:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949341.109074:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:1.0:1784949341.109076:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000400:00000200:1.0:1784949341.109080:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-110 00000400:00000200:1.0:1784949341.109082:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -110 00000400:00000200:1.0:1784949341.109085:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00010000:00000001:2.0:1784949341.428023:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949341.428033:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949341.428040:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949341.428042:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949341.428215:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949341.428222:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949341.428227:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1784949341.620088:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949341.620103:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949341.620113:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005126bdfa. 00000020:00000040:1.0:1784949341.620157:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949341.620161:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949341.620166:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949341.620173:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949341.620173:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1784949341.620184:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949341.620197:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000c53910d9. 00000100:00100000:2.0:1784949341.620199:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949316/real 1784949316] req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949341 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949341.620209:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949341.620225:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1784949341.620245:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784949341.620246:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1784949341.620255:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1784949341.620256:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949341.620267:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000384f0ed7. 00000100:00000040:1.0:1784949341.620270:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005126bdfa] to pc [ptlrpcd_00_03+3] req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949341.620283:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000588d3b65 00000400:00000010:2.0:1784949341.620292:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000588d3b65. 00000100:00000001:2.0:1784949341.620302:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784949341.620310:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949341.620313:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949341 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1784949341.620317:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:1.0:1784949341.620329:0:335217:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1784949341.620337:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949341.620351:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1784949341.620360:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000029a32fda. 00000100:00000200:2.0:1784949341.620363:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949341 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949341.620367:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:1.0:1784949341.620376:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1784949341.620380:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1784949341.620385:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949341.620397:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000001ee120ca. 00000100:00000001:2.0:1784949341.620398:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949341.620407:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.620409:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949341.620418:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949341.620420:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949341.620427:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949341.620440:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949341.620444:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949341.620447:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949341 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949341.620456:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029a32fda] to pc [ptlrpcd_00_00+0] req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949341.620462:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949341.620477:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:3.0:1784949341.620488:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949341.620499:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.620507:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949341.620513:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949341.620514:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949341.620519:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949341.620521:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.620522:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949341.620525:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949341.620533:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949341.620538:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949341.620539:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11618) 00000100:00000001:2.0:1784949341.620546:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1784949341.620546:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949341.620556:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949341.620563:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949341.620571:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000040:2.0:1784949341.620577:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949341 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949341.620580:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949341.620581:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.620588:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.620596:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784949341.620600:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949341.620604:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949341.620607:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949341.620615:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949341.620618:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.620624:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949341.620625:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1784949341.620627:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289690496:0@lo:400: 00000100:00000001:1.0:1784949341.620634:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949341.620640:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.620644:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949341.620647:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1784949341.620653:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949341.620654:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1784949341.620655:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949341.620664:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.620668:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1784949341.620676:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949341.620679:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949341.620693:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1784949341.620701:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949341.620708:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949341.620715:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949341.620720:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949341.620728:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949341.620729:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:2.0:1784949341.620741:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1784949341.620744:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289690624:192.168.202.137@tcp:400: 00000100:00080000:0.0:1784949341.620754:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000040:2.0:1784949341.620759:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949341 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949341.620760:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1784949341.620768:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1784949341.620772:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949341.620774:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1784949341.620780:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949341.620781:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1784949341.620783:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f37d62c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289689216:192.168.202.137@tcp:8: 02000000:00000001:1.0:1784949341.620792:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949341.620794:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1784949341.620796:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949341.620801:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1784949341.620802:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000000013c6c1. 00000100:00080000:0.0:1784949341.620804:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11588 00000100:00000040:2.0:1784949341.620808:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f37d62c3 x1871651289689216/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949341 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949341.620813:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949341.620825:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000400:00000010:1.0:1784949341.620831:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000035db9be3. 00000100:00000001:2.0:1784949341.620831:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1784949341.620834:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11587 00000400:00000010:1.0:1784949341.620846:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000400:00000200:0.0:1784949341.620848:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000200:1.0:1784949341.620857:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289690624, portal 10 00000100:00000040:0.0:1784949341.620859:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949341.620866:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949341.620870:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949341.620871:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637809152 : -115544071742464 : ffff96e9cca5dc00) 00000100:00000040:0.0:1784949341.620881:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949341.620887:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000100:00000040:1.0:1784949341.620891:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949341.620897:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949341.620943:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:1.0:1784949341.620946:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1784949341.620955:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1784949341.620955:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949341.620960:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000020:00000040:0.0:1784949341.620964:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000100:00000200:1.0:1784949341.620966:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289690624, offset 0 00000020:00000001:0.0:1784949341.620975:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000200:1.0:1784949341.620983:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949341.621006:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:0.0:1784949341.621016:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949341.621021:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784949341.621024:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000040:0.0:1784949341.621031:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000200:1.0:1784949341.621035:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784949341.621038:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:1.0:1784949341.621045:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000040:0.0:1784949341.621048:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000400:00000200:1.0:1784949341.621054:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1784949341.621079:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000400:00000200:1.0:1784949341.621080:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00080000:0.0:1784949341.621095:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000200:1.0:1784949341.621097:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3921:3921:240:4294967295] 192.168.202.137@tcp LPNI seq info [2502:2502:8:4294967295] 00000100:00000001:0.0:1784949341.621107:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949341.621109:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1784949341.621115:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1784949341.621116:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009e9a21e3. 02000000:00000001:2.0:1784949341.621147:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949341.621154:0:337118:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:0.0:1784949341.621155:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949341.621157:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000040:2.0:1784949341.621159:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784949341.621162:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.621171:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000020:00000001:2.0:1784949341.621171:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:0.0:1784949341.621177:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949341.621177:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 0000000068fddc2b. 00000800:00000200:1.0:1784949341.621183:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000010:0.0:1784949341.621190:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004614f9cb. 02000000:00000001:2.0:1784949341.621190:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000800:00000010:1.0:1784949341.621193:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a934b7fa. 02000000:00000001:2.0:1784949341.621196:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:0.0:1784949341.621197:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000010:2.0:1784949341.621203:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f37d62c3. 02000000:00000001:0.0:1784949341.621205:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.0:1784949341.621212:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 02000000:00000001:0.0:1784949341.621213:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.621214:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949341.621220:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949341.621225:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949341.621225:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949341.621229:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949341.621232:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1784949341.621234:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949341.621235:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784949341.621238:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949341.621243:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a934b7fa type 1, nob 320 niov 1 nkiov 1 02000000:00000010:0.0:1784949341.621245:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000003bbd92b2. 00000100:00000001:2.0:1784949341.621248:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949341.621255:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.621259:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.621271:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949341.621273:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1784949341.621282:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2253 usecs (3 jiffies) 00000100:00000001:1.0:1784949341.621293:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306280964 00000100:00000001:0.0:1784949341.621306:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:1.0:1784949341.621307:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000054c9929b. 00000100:00000001:0.0:1784949341.621314:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.621314:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949341.621321:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.621322:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000040:0.0:1784949341.621325:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004614f9cb] to pc [ptlrpcd_rcv+-1] req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949341.621337:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306280964 02000000:00000010:2.0:1784949341.621347:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001c9428b0. 00000100:00000010:1.0:1784949341.621351:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a6e97b61. 02000000:00000001:2.0:1784949341.621354:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.621361:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949341.621362:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949341.621369:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a16404b8. 00000100:00000001:1.0:1784949341.621372:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306280964 00000100:00000040:0.0:1784949341.621377:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11618) 00000100:00000010:1.0:1784949341.621382:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000400:00000010:2.0:1784949341.621383:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f88d5b9. 00000100:00000001:1.0:1784949341.621394:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000200:2.0:1784949341.621398:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289690496, portal 25 00000100:00000001:1.0:1784949341.621402:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306280964 00000100:00000001:2.0:1784949341.621408:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000010:1.0:1784949341.621413:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:2.0:1784949341.621415:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529637815424 : -115544071736192 : ffff96e9cca5f480) 00000100:00000001:1.0:1784949341.621421:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949341.621430:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949341.621433:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949341.621435:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949341.621440:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949341.621444:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949341.621449:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.621455:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.621473:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949341.621482:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949341.621488:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000200:2.0:1784949341.621495:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289690496, offset 0 00000400:00000200:2.0:1784949341.621509:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949341.621526:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949341.621528:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949341.621534:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:0.0:1784949341.621540:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949341.621540:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000800:00000010:0.0:1784949341.621549:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a934b7fa. 00000400:00000200:2.0:1784949341.621550:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f180 00000800:00000001:1.0:1784949341.621551:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949341.621563:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949341.621566:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8d15 [2] + 0 00000800:00000001:1.0:1784949341.621571:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949341.621573:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001d3d7473. 00000400:00000200:0.0:1784949341.621581:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949341.621582:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949341.621588:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000fe8db63d 00000800:00000001:1.0:1784949341.621593:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949341.621596:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000400:00000200:2.0:1784949341.621599:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1784949341.621605:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949341.621606:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000400:00000200:1.0:1784949341.621615:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949341.621617:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949341.621620:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949341.621636:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949341.621637:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fe8db63d 00000400:00000010:2.0:1784949341.621646:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000fe8db63d. 00000100:00000001:3.0:1784949341.621648:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784949341.621655:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f200 00000100:00000001:3.0:1784949341.621656:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.621659:0:337121:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:3.0:1784949341.621668:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000200:2.0:1784949341.621669:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 02000000:00000001:3.0:1784949341.621675:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949341.621680:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949341.621680:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000a73b72ec x1871651289690496 msgsize 224 02000000:00000001:3.0:1784949341.621684:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949341.621689:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 181808 00000100:00000040:0.0:1784949341.621694:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949341.621701:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289690752:1.2.3.4@tcp:8: 00000100:00100000:2.0:1784949341.621708:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000001:1.0:1784949341.621719:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949341.621720:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949341.621722:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949341.621731:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.621733:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949341.621736:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949341.621738:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949341.621742:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949341.621744:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.621747:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784949341.621750:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949341.621751:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949341.621764:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000100:00000001:3.0:1784949341.621767:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784949341.621775:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:2.0:1784949341.621777:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:3.0:1784949341.621777:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.621785:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949341.621790:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949341.621791:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1784949341.621798:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784949341.621801:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:2.0:1784949341.621803:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949341.621804:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00100000:0.0:1784949341.621810:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289690496 02000000:00000001:0.0:1784949341.621818:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000010:1.0:1784949341.621818:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000cd114eaa. 00000100:00000001:0.0:1784949341.621826:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:1.0:1784949341.621830:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000cd114eaa x1871651289690624 msgsize 224 00000100:00000001:0.0:1784949341.621832:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.621832:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949341.621842:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949341.621846:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949341.621851:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289690496 00000100:00100000:1.0:1784949341.621852:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000020:00000001:0.0:1784949341.621860:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949341.621860:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949341.621866:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949341.621871:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949341.621873:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949341.621879:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000040:0.0:1784949341.621881:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949341.621881:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949341.621888:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949341.621888:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1784949341.621894:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000001:1.0:1784949341.621894:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949341.621900:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949341.621905:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949341.621950:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306281004 00000100:00000001:0.0:1784949341.621958:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949341.621967:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:2.0:1784949341.621967:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306280964 00000100:00000001:0.0:1784949341.621973:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949341.621977:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306281004 00000100:00000001:1.0:1784949341.621978:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000020:00000010:0.0:1784949341.621980:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00100000:1.0:1784949341.621988:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289690624 00000100:00000001:2.0:1784949341.621991:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306280964 02000000:00000001:1.0:1784949341.621995:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000010:0.0:1784949341.621997:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004565e281. 00000100:00000001:1.0:1784949341.622001:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949341.622002:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306281004 00000100:00000001:1.0:1784949341.622008:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949341.622013:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 02000000:00000001:1.0:1784949341.622015:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.622016:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306280964 00000020:00000010:0.0:1784949341.622026:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000027aee7d. 00000100:00100000:1.0:1784949341.622027:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289690624 00000100:00000001:2.0:1784949341.622031:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306281004 00000020:00000001:1.0:1784949341.622035:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949341.622041:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000100:00000001:2.0:1784949341.622043:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306280964 00000020:00000001:1.0:1784949341.622045:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1784949341.622046:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000020:00000040:1.0:1784949341.622054:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:0.0:1784949341.622056:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949341.622056:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1784949341.622062:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949341.622064:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949341.622064:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949341.622069:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949341.622072:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949341.622072:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.622076:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949341.622082:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949341.622085:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:0.0:1784949341.622088:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.622094:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949341.622099:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000020:00000010:1.0:1784949341.622109:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000100:00000001:0.0:1784949341.622112:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1784949341.622115:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009adcfc93. 00000020:00000010:1.0:1784949341.622149:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949341.622170:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949341.622171:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949341.622174:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949341.622179:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949341.622180:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949341.622187:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949341.622195:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 831 00000100:00000001:1.0:1784949341.622198:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000040:0.0:1784949341.622202:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949341.622203:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949341.622208:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.622212:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534180532304 : -115539529019312 : ffff96eadb6a5050) 00000100:00000040:0.0:1784949341.622227:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000a73b72ec x1871651289690496/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:247/0 lens 224/0 e 0 to 0 dl 1784949352 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949341.622228:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949341.622240:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949341.622246:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949341.622260:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:1.0:1784949341.622261:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 807 00000100:00000001:3.0:1784949341.622262:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949341.622271:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000010:0.0:1784949341.622273:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000040:1.0:1784949341.622274:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949341.622276:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.622284:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637804672 : -115544071746944 : ffff96e9cca5ca80) 00000100:00000001:3.0:1784949341.622287:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.622288:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949341.622298:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000cd114eaa x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:247/0 lens 224/0 e 0 to 0 dl 1784949352 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949341.622303:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000001:1.0:1784949341.622322:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:0.0:1784949341.622325:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000a73b72ec pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289690496:12345-0@lo:400: 00000020:00000010:1.0:1784949341.622330:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000200:0.0:1784949341.622342:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289690496 00000100:00000001:1.0:1784949341.622343:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949341.622352:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000020:00000001:0.0:1784949341.622355:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949341.622365:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00100000:1.0:1784949341.622365:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000cd114eaa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289690624:12345-192.168.202.137@tcp:400: 00000020:00000001:0.0:1784949341.622375:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949341.622375:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289690624 00000020:00000001:1.0:1784949341.622382:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949341.622385:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949341.622390:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949341.622391:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:1.0:1784949341.622395:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949341.622404:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949341.622404:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949341.622415:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949341.622415:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949341.622427:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949341.622429:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949341.622433:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949341.622437:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949341.622439:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949341.622446:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949341.622447:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949341.622449:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949341.622459:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000001:0.0:1784949341.622461:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949341.622466:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949341.622469:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949341.622472:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:0.0:1784949341.622477:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949341.622480:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000009e9a21e3. 02000000:00000001:1.0:1784949341.622486:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949341.622488:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000b1427c95. 00000100:00000001:1.0:1784949341.622490:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949341.622497:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949341.622500:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949341.622502:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289690624 00000100:00000001:0.0:1784949341.622511:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949341.622511:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949341.622520:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000cd114eaa x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:247/0 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949341.622522:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949341.622534:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289690496 00010000:00000001:1.0:1784949341.622539:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949341.622545:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000001:1.0:1784949341.622546:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949341.622554:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=58 v=5 (1 1 1 1) 00010000:00000200:0.0:1784949341.622559:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000a73b72ec x1871651289690496/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:247/0 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949341.622563:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949341.622572:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949341.622580:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949341.622591:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00010000:00000001:0.0:1784949341.622592:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:1.0:1784949341.622598:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949341.622599:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949341.622607:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00001000:0.0:1784949341.622608:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=63 v=5 (1 1 1 1) 00000100:00000040:1.0:1784949341.622617:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949341.622619:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1784949341.622624:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000100:00000040:0.0:1784949341.622626:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949341.622636:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000200:1.0:1784949341.622639:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289690624, offset 224 02000000:00000001:0.0:1784949341.622641:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949341.622645:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.622652:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949341.622663:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:1.0:1784949341.622664:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949341.622670:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00000200:0.0:1784949341.622679:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289690496, offset 224 00000400:00000200:1.0:1784949341.622687:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949341.622695:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949341.622709:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3922:3922:240:4294967295] 192.168.202.137@tcp LPNI seq info [2503:2503:8:4294967295] 00000400:00000200:1.0:1784949341.622726:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949341.622734:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949341.622741:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000cf155122. 00000400:00000200:0.0:1784949341.622742:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000800:00000200:1.0:1784949341.622754:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:0.0:1784949341.622762:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f180 00000800:00000200:1.0:1784949341.622770:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000200:0.0:1784949341.622778:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x971d [1] + 224 00000800:00000200:1.0:1784949341.622779:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000cf155122 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1784949341.622796:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949341.622803:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:1.0:1784949341.622809:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.622812:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1784949341.622819:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000100:00000200:0.0:1784949341.622826:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949341.622827:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949341.622833:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949341.622841:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949341.622854:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000cd114eaa x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:247/0 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949341.622869:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949341.622880:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000cd114eaa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289690624:12345-192.168.202.137@tcp:400: Request processed in 552us (1044us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949341.622941:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 807 00000100:00000040:1.0:1784949341.622950:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949341.622955:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949341.622958:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949341.622961:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:1.0:1784949341.622965:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009adcfc93. 00000020:00000010:1.0:1784949341.622973:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000400:00000200:0.0:1784949341.622977:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000020:00000010:1.0:1784949341.622980:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000400:00000010:0.0:1784949341.622986:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000020:00000040:1.0:1784949341.622990:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949341.622997:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949341.622998:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949341.623007:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949341.623020:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949341.623027:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784949341.623032:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949341.623037:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949341.623042:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000800:00000200:2.0:1784949341.623043:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949341.623047:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:2.0:1784949341.623050:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000cf155122. 00000800:00000001:1.0:1784949341.623052:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:0.0:1784949341.623055:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a73b72ec x1871651289690496/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:247/0 lens 224/224 e 0 to 0 dl 1784949352 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949341.623055:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949341.623061:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949341.623066:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1784949341.623068:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949341.623076:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949341.623078:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949341.623079:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:3.0:1784949341.623083:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000a16404b8. 00000800:00000001:1.0:1784949341.623089:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1784949341.623096:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000a73b72ec pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289690496:12345-0@lo:400: Request processed in 831us (1409us total) trans 0 rc 0/0 00000400:00000200:3.0:1784949341.623097:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f88d5b9 00000400:00000010:3.0:1784949341.623105:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f88d5b9. 00000400:00000200:1.0:1784949341.623111:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949341.623116:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1784949341.623144:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:2.0:1784949341.623153:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000100:00100000:0.0:1784949341.623154:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 831 00000100:00000040:0.0:1784949341.623162:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949341.623163:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000200:3.0:1784949341.623163:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949341.623164:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f200 00000100:00000001:0.0:1784949341.623172:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949341.623172:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949341.623182:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000200:1.0:1784949341.623190:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9715 [1] + 224 00000020:00000010:0.0:1784949341.623194:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000020:00000010:0.0:1784949341.623210:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000027aee7d. 00000800:00000001:1.0:1784949341.623212:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:3.0:1784949341.623215:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949341.623224:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004565e281. 00000800:00000001:1.0:1784949341.623228:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949341.623239:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949341.623240:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:0.0:1784949341.623243:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949341.623248:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949341.623255:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949341.623258:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949341.623260:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:3.0:1784949341.623266:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949341.623267:0:337113:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:0.0:1784949341.623268:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000b1427c95. 00000100:00000001:3.0:1784949341.623274:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949341.623277:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949341.623278:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949341.623280:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949341.623286:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949341.623288:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949341.623301:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:3.0:1784949341.623302:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=63 v=5 (5 5 5 5) 00000400:00000010:0.0:1784949341.623314:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c7491352. 00000100:00001000:3.0:1784949341.623317:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=63 v=5 (1 1 1 1) 00000100:00000040:1.0:1784949341.623321:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949341.623327:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001c210c55. 00000100:00000001:3.0:1784949341.623330:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949341.623339:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949341.623347:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949341.623353:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949341.623364:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949341.623379:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949341.623392:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949341.623394:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949341.623396:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949341.623400:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949341.623402:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1784949341.623414:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000005126bdfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289690496:0@lo:400: 00000800:00000001:1.0:1784949341.623415:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949341.623426:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.623428:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.623435:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949341.623440:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000005126bdfa x1871651289690496/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949341.623448:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949341.623450:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1784949341.623456:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1784949341.623458:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949341.623465:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001c9428b0. 00000400:00000001:0.0:1784949341.623468:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1784949341.623471:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000001:0.0:1784949341.623474:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1784949341.623477:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000010:0.0:1784949341.623480:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000035db9be3. 00000020:00000040:3.0:1784949341.623482:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:0.0:1784949341.623490:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000041a6162e 00000020:00000001:3.0:1784949341.623491:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949341.623496:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000c53910d9. 00000400:00000010:0.0:1784949341.623498:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000041a6162e. 02000000:00000001:3.0:1784949341.623508:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1784949341.623510:0:337120:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1784949341.623513:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949341.623517:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005126bdfa. 00000100:00000200:0.0:1784949341.623522:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949341.623526:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949341.623532:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.623540:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949341.623552:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:0.0:1784949341.623554:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949341.623556:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949341.623567:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949341.623573:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.623585:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949341.623588:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949341.623591:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949341.623593:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949341.623596:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949341.623599:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949341.623602:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949341.623609:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=58 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949341.623613:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=58 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949341.623616:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949341.623618:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949341.623620:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949341.623625:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949341.623634:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.623637:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949341.623639:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949341.623641:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949341.623643:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949341.623646:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949341.623655:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949341.623658:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949341.623659:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949341.623662:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949341.623671:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000029a32fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289690624:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949341.623676:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949341.623678:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949341.623680:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000029a32fda x1871651289690624/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949357 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949341.623688:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949341.623690:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949341.623693:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000000013c6c1. 02000000:00000001:0.0:1784949341.623698:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949341.623700:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949341.623702:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949341.623704:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949341.623706:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000001ee120ca. 02000000:00000001:0.0:1784949341.623709:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949341.623711:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949341.623713:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000029a32fda. 00000100:00000001:0.0:1784949341.623716:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949341.623718:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949341.623720:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949341.623724:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949341.623726:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949341.623728:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949341.623731:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949341.664996:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306281004, at 4306281005 00000100:00000010:3.0:1784949341.665011:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000040:0.0:1784949341.665052:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306281004, at 4306281005 00000100:00000040:2.0:1784949341.665062:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306281004, at 4306281005 00000100:00000010:0.0:1784949341.665072:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000054c9929b. 00000100:00000040:1.0:1784949341.665072:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306281004, at 4306281005 00000100:00000010:2.0:1784949341.665072:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000010:1.0:1784949341.665092:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a6e97b61. 00010000:00000001:2.0:1784949342.452058:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949342.452071:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949342.452083:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949342.452088:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949342.452092:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949342.452097:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949342.452101:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949342.452103:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949342.452106:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949342.452109:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949342.452264:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949342.452271:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949342.452275:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949342.644097:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949342.644107:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1784949342.644148:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949341/real 0] req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949342 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.644169:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.644174:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949342.644178:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949342.644186:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949342 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.644202:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.644216:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949342.644220:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.644225:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949342.644230:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949342.644234:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949342.644242:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949342 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.644257:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1784949342.644400:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.644405:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949342.644408:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.644414:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1784949342.644419:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1784949342.644426:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1784949342.644464:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:3.0:1784949342.644470:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1784949342.644480:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949342 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949342.644499:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289690752:1.2.3.4@tcp:8: 00000100:00000001:3.0:1784949342.644513:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.644519:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949342.644524:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004614f9cb x1871651289690752/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949342 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949342.644528:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949342.644545:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949342.644554:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003bf8e8ad. 00000020:00000040:0.0:1784949342.644564:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949342.644574:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1784949342.644578:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949342.644581:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1784949342.644587:0:337118:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:0.0:1784949342.644590:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949342.644594:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:3.0:1784949342.644603:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949342.644604:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000353a5c12. 02000000:00000010:3.0:1784949342.644610:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000003bbd92b2. 00000100:00000001:0.0:1784949342.644615:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949342.644624:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:0.0:1784949342.644628:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1784949342.644632:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949342.644639:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004614f9cb. 00000100:00000001:3.0:1784949342.644651:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949342.644656:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949342.644662:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949342.644664:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949342.644670:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.644677:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949342.644683:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949342.644687:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf8e8ad] to pc [ptlrpcd_00_01+1] req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.644691:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949342.644698:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949342.644751:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949342.644758:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949342.644765:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949342.644771:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949342.644775:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949342.644781:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1784949342.644788:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784949342.644790:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11588 00000100:00100000:3.0:1784949342.644799:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:3.0:1784949342.644806:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949342.644813:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:3.0:1784949342.644817:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949342.644822:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1784949342.644826:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11613 00000100:00000001:3.0:1784949342.644828:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949342.644833:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949342.644839:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1784949342.644846:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -114 00000100:00000040:3.0:1784949342.644848:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949342.644852:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1784949342.644858:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1784949342.644862:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949342.644876:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:3.0:1784949342.644882:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949342.644888:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949342.644889:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1784949342.644894:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949342.644901:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:3.0:1784949342.644901:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949342.644937:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949342.644942:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289690880:0@lo:400: 00000100:00000001:3.0:1784949342.644949:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949342.644956:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949342.644960:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1784949342.644963:0:335217:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:3.0:1784949342.644966:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949342.644968:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 02000000:00000001:3.0:1784949342.644972:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1784949342.644974:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949342.644982:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000010:3.0:1784949342.644984:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 00000020:00000001:0.0:1784949342.644986:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:3.0:1784949342.644992:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949342.644995:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:3.0:1784949342.645001:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000100:00000040:0.0:1784949342.645003:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949342.645010:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949342.645013:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000400:00000010:3.0:1784949342.645014:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00000001:0.0:1784949342.645016:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:0.0:1784949342.645024:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000200:3.0:1784949342.645028:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289690880, portal 25 00000100:00080000:0.0:1784949342.645031:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00000001:3.0:1784949342.645036:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949342.645041:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469562752 : -115544239988864 : ffff96e9c29e9f80) 00000100:00080000:0.0:1784949342.645046:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000040:3.0:1784949342.645053:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949342.645057:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949342.645069:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1784949342.645074:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949342.645080:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949342.645083:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 02000000:00000001:0.0:1784949342.645085:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1784949342.645087:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289690880, offset 0 02000000:00000001:0.0:1784949342.645098:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949342.645104:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:0.0:1784949342.645137:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1784949342.645145:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949342.645152:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000010:0.0:1784949342.645156:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000aedb1960. 00000020:00000040:0.0:1784949342.645166:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:3.0:1784949342.645173:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f300 02000000:00000001:0.0:1784949342.645178:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949342.645184:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949342.645189:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8d2d [2] + 0 00000100:00000001:0.0:1784949342.645194:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1784949342.645197:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000603fb577. 02000000:00000001:0.0:1784949342.645201:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1784949342.645205:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000031b88578 02000000:00000001:0.0:1784949342.645210:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949342.645218:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:0.0:1784949342.645224:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000b1427c95. 00000400:00000200:3.0:1784949342.645227:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949342.645241:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949342.645241:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031b88578 00000400:00000010:3.0:1784949342.645247:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000031b88578. 00000100:00000001:3.0:1784949342.645256:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949342.645259:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00100000:0.0:1784949342.645263:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1784949342.645267:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000222ac126 x1871651289690880 msgsize 224 00000100:00100000:3.0:1784949342.645284:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:3.0:1784949342.645293:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949342.645307:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949342.645310:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.645315:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:0.0:1784949342.645321:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aedb1960] to pc [ptlrpcd_rcv+-1] req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949342.645329:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000400:00000010:3.0:1784949342.645336:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000100:00000001:3.0:1784949342.645343:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949342.645353:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.645374:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949342.645379:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784949342.645380:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949342.645381:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949342.645388:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289690880 00000100:00000040:0.0:1784949342.645395:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1784949342.645396:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949342.645405:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949342.645410:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.645419:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949342.645423:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:2.0:1784949342.645426:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.645426:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:0.0:1784949342.645434:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eac1e2f5. 00000100:00100000:2.0:1784949342.645436:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289690880 00000100:00000001:3.0:1784949342.645439:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949342.645445:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949342.645446:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000040:2.0:1784949342.645451:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:3.0:1784949342.645451:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306282027 00000100:00000001:0.0:1784949342.645458:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1784949342.645459:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949342.645467:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:3.0:1784949342.645467:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 02000000:00000001:0.0:1784949342.645468:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949342.645476:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949342.645479:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949342.645487:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784949342.645488:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949342.645488:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 02000000:00000010:0.0:1784949342.645492:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000d104522b. 00000100:00000001:1.0:1784949342.645496:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949342.645501:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949342.645503:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949342.645508:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.645509:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306282027 00000020:00000010:2.0:1784949342.645514:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000040:1.0:1784949342.645518:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949342.645527:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1784949342.645528:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000040:0.0:1784949342.645530:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:2.0:1784949342.645544:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00000001:3.0:1784949342.645545:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949342.645562:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306282027 00000100:00000001:1.0:1784949342.645563:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000010:2.0:1784949342.645567:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001c9428b0. 00000100:00000001:1.0:1784949342.645569:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949342.645579:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:0.0:1784949342.645581:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949342.645582:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000010:2.0:1784949342.645588:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004268a481. 00000100:00000001:0.0:1784949342.645593:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949342.645593:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949342.645595:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:2.0:1784949342.645601:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 02000000:00000001:1.0:1784949342.645608:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.645610:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306282027 00000100:00000040:0.0:1784949342.645619:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eac1e2f5] to pc [ptlrpcd_00_02+2] req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949342.645620:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289691008:192.168.202.137@tcp:8: 00000100:00000010:3.0:1784949342.645622:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00000040:2.0:1784949342.645624:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949342.645636:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949342.645637:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949342.645638:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949342.645645:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:1.0:1784949342.645649:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949342.645651:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949342.645652:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949342.645657:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.645666:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949342.645668:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1784949342.645673:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.645679:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949342.645681:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000004cb3d562. 00000100:00000001:3.0:1784949342.645688:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949342.645689:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949342.645702:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949342.645704:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000877cac20. 00000400:00000010:1.0:1784949342.645715:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000100:00000001:0.0:1784949342.645722:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949342.645722:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289691008, portal 4 00000100:00000001:3.0:1784949342.645723:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949342.645725:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:3.0:1784949342.645732:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949342.645734:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949342.645735:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949342.645739:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949342.645741:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11619) 00000100:00000001:1.0:1784949342.645742:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469559168 : -115544239992448 : ffff96e9c29e9180) 00000100:00100000:2.0:1784949342.645752:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 832 00000100:00000040:3.0:1784949342.645753:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949342.645765:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949367 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949342.645765:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949342.645777:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165125200 : -115539544426416 : ffff96eada7f3850) 00000100:00000040:2.0:1784949342.645794:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000222ac126 x1871651289690880/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:248/0 lens 224/0 e 0 to 0 dl 1784949353 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.645800:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949342.645806:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949342.645819:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949342.645824:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949342.645825:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949342.645828:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949342.645836:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949342.645839:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949342.645845:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949342.645846:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00100000:3.0:1784949342.645854:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eac1e2f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289691136:192.168.202.137@tcp:400: 00000020:00000010:2.0:1784949342.645857:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000400:00000010:1.0:1784949342.645858:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000100:00000001:3.0:1784949342.645866:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949342.645868:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1784949342.645871:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289691008, offset 0 02000000:00000001:3.0:1784949342.645874:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000020:00000010:2.0:1784949342.645875:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 02000000:00000001:3.0:1784949342.645882:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949342.645888:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1784949342.645895:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000222ac126 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289690880:12345-0@lo:400: 02000000:00000010:3.0:1784949342.645896:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 00000400:00000200:1.0:1784949342.645898:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:3.0:1784949342.645944:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949342.645945:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289690880 00000020:00000001:2.0:1784949342.645957:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000010:3.0:1784949342.645959:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000084252b45. 00000020:00000001:2.0:1784949342.645969:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000010:3.0:1784949342.645969:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 00000400:00000200:1.0:1784949342.645970:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000001:2.0:1784949342.645979:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949342.645983:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289691136, portal 10 00000400:00000200:1.0:1784949342.645988:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1784949342.645989:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:3.0:1784949342.645991:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949342.645998:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469569920 : -115544239981696 : ffff96e9c29ebb80) 00000020:00000001:2.0:1784949342.646001:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000200:1.0:1784949342.646005:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:3.0:1784949342.646013:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949342.646023:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000400:00000200:1.0:1784949342.646029:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1784949342.646032:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.646044:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1784949342.646045:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000001:2.0:1784949342.646046:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:3.0:1784949342.646054:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784949342.646056:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:3.0:1784949342.646059:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000821d7a6f. 00000020:00000001:2.0:1784949342.646065:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949342.646068:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3923:3923:240:4294967295] 192.168.202.137@tcp LPNI seq info [2504:2504:8:4294967295] 00000100:00000200:3.0:1784949342.646069:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289691136, offset 0 00000020:00000001:2.0:1784949342.646072:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000200:3.0:1784949342.646086:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949342.646087:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949342.646097:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949342.646105:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d58187af. 02000000:00000001:2.0:1784949342.646139:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949342.646144:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949342.646151:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949342.646156:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00010000:00000040:2.0:1784949342.646158:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289690880 00010000:00000001:2.0:1784949342.646166:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949342.646179:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000222ac126 x1871651289690880/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:248/0 lens 224/224 e 0 to 0 dl 1784949353 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949342.646185:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949342.646188:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000800:00000010:1.0:1784949342.646203:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a21d8ef3. 00000400:00000200:3.0:1784949342.646207:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1784949342.646211:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949342.646217:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949342.646218:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000800:00000200:1.0:1784949342.646226:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00001000:2.0:1784949342.646229:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=64 v=5 (1 1 1 1) 00000400:00000200:3.0:1784949342.646234:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784949342.646244:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1784949342.646245:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:2.0:1784949342.646255:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000800:00000200:1.0:1784949342.646256:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:3.0:1784949342.646263:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3924:3924:239:4294967295] 192.168.202.137@tcp LPNI seq info [2505:2505:7:4294967295] 00000100:00000001:2.0:1784949342.646267:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000200:1.0:1784949342.646269:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a21d8ef3 type 1, nob 616 niov 1 nkiov 1 02000000:00000001:2.0:1784949342.646280:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949342.646284:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949342.646290:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949342.646303:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784949342.646306:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949342.646308:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000400:00000010:2.0:1784949342.646313:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000040:1.0:1784949342.646316:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000100:00000200:2.0:1784949342.646324:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289690880, offset 224 00000800:00000200:3.0:1784949342.646326:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949342.646327:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306282028 00000400:00000200:2.0:1784949342.646338:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000010:3.0:1784949342.646342:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000001:1.0:1784949342.646344:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4306282027 00000400:00000200:2.0:1784949342.646358:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949342.646359:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306282028 00000800:00000200:3.0:1784949342.646367:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949342.646373:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4306282027 00000800:00000200:3.0:1784949342.646392:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949342.646395:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306282028 00000400:00000200:2.0:1784949342.646400:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f300 00000800:00000200:3.0:1784949342.646403:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949342.646408:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4306282027 00000400:00000200:2.0:1784949342.646419:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9731 [1] + 224 00000100:00000001:1.0:1784949342.646422:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306282028 00000400:00000200:2.0:1784949342.646431:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949342.646432:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949342.646433:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4306282027 00000400:00000200:2.0:1784949342.646447:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:3.0:1784949342.646449:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2256 usecs (3 jiffies) 00000100:00000001:1.0:1784949342.646450:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949342.646454:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1784949342.646460:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.646465:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306281989 00000100:00000200:2.0:1784949342.646466:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949342.646468:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949342.646480:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.646481:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306282028 00000100:00000001:1.0:1784949342.646493:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949342.646497:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306281989 00000100:00000001:1.0:1784949342.646500:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000040:2.0:1784949342.646504:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.646510:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306282028 00000100:00000001:1.0:1784949342.646512:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949342.646518:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949342.646526:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306281989 00000100:00000001:1.0:1784949342.646527:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1784949342.646529:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949342.646538:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.646540:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306282028 00000800:00000010:0.0:1784949342.646541:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a21d8ef3. 00000100:00000001:2.0:1784949342.646549:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949342.646554:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306281989 00000400:00000200:0.0:1784949342.646567:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949342.646568:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306282028 00000800:00000001:1.0:1784949342.646570:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949342.646573:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000100:00000001:3.0:1784949342.646581:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949342.646582:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000800:00000001:1.0:1784949342.646589:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949342.646592:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949342.646595:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949342.646597:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949342.646598:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949342.646609:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949342.646610:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949342.646612:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003f05b564 00000800:00000001:1.0:1784949342.646613:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.646620:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949342.646623:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003f05b564. 00000100:00000001:2.0:1784949342.646623:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.646631:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784949342.646636:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949342.646638:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000040:2.0:1784949342.646641:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:0.0:1784949342.646642:0:337114:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:2.0:1784949342.646653:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949342.646654:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1784949342.646657:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.646661:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:0.0:1784949342.646663:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949367 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949342.646666:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643f380 00000020:00000001:2.0:1784949342.646670:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784949342.646671:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.646685:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949342.646686:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000222ac126 x1871651289690880/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:248/0 lens 224/224 e 0 to 0 dl 1784949353 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000100:1.0:1784949342.646695:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289691008 offset 0 length 520: 4 00000100:00000001:3.0:1784949342.646711:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1784949342.646713:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000279db1a3 is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:3.0:1784949342.646719:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1784949342.646727:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000400:00000001:3.0:1784949342.646733:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784949342.646735:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000001:3.0:1784949342.646740:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949342.646745:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000800:00000001:1.0:1784949342.646747:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784949342.646750:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000222ac126 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289690880:12345-0@lo:400: Request processed in 901us (1477us total) trans 0 rc 0/0 00000100:00000040:0.0:1784949342.646755:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949367 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949342.646759:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949342.646760:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:3.0:1784949342.646769:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:3.0:1784949342.646779:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949342.646783:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 832 00000100:00000200:3.0:1784949342.646791:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949342.646796:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949342.646807:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949342.646814:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949342.646816:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949342.646824:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004268a481. 00000100:00000001:0.0:1784949342.646827:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:3.0:1784949342.646830:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949342.646832:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949342.646843:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001c9428b0. 00000800:00000001:0.0:1784949342.646849:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949342.646856:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1784949342.646857:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949342.646863:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949342.646867:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949342.646869:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000010:2.0:1784949342.646872:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d58187af. 02000000:00000001:3.0:1784949342.646875:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949342.646880:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949342.646882:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949342.646892:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:3.0:1784949342.646894:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949342.646895:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949342.646898:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784949342.646905:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000400:00000200:1.0:1784949342.646944:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00001000:3.0:1784949342.646946:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=64 v=5 (5 5 5 5) 00000800:00000001:0.0:1784949342.646951:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949342.646954:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:3.0:1784949342.646955:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=64 v=5 (1 1 1 1) 00000400:00000200:1.0:1784949342.646962:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:0.0:1784949342.646966:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949342.646969:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:2.0:1784949342.646972:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001d3d7473. 00000400:00000200:1.0:1784949342.646974:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000821d7a6f 00000100:00000001:3.0:1784949342.646976:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949342.646984:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000821d7a6f. 00010000:00000001:3.0:1784949342.646985:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000800:00000001:0.0:1784949342.646987:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949342.646993:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949342.646995:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:0.0:1784949342.646999:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949342.647000:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949342.647002:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004089dc8d. 00000100:00000200:1.0:1784949342.647008:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949342.647013:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949342.647018:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949342.647046:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949342.647055:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1784949342.647067:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949342.647069:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.647080:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:0.0:1784949342.647086:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f400 00000100:00000001:3.0:1784949342.647086:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949342.647100:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949342.647102:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949342.647107:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1784949342.647143:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 182032 00000100:00100000:3.0:1784949342.647161:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289690880:0@lo:400: 00000800:00000001:0.0:1784949342.647167:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949342.647169:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.647176:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949342.647179:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003bf8e8ad x1871651289690880/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.647190:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:0.0:1784949342.647191:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949342.647193:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949342.647199:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 00000400:00000200:0.0:1784949342.647202:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784949342.647210:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949342.647216:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:0.0:1784949342.647218:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000040:3.0:1784949342.647220:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949342.647228:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949342.647234:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000353a5c12. 02000000:00000001:3.0:1784949342.647238:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1784949342.647240:0:337114:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:3.0:1784949342.647241:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949342.647248:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003bf8e8ad. 00000100:00000200:0.0:1784949342.647249:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784949342.647253:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949342.647257:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.647260:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1784949342.647261:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000925b9f50. 00000100:00000001:3.0:1784949342.647265:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949342.647272:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949342.647273:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000925b9f50 x1871651289691136 msgsize 224 00000100:00000001:3.0:1784949342.647276:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949342.647279:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949342.647293:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949342.647324:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000800:00000001:0.0:1784949342.647336:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949342.647343:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949342.647383:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:1.0:1784949342.647387:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289691136 02000000:00000001:1.0:1784949342.647390:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949342.647392:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949342.647394:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949342.647398:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949342.647401:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289691136 00000020:00000001:1.0:1784949342.647403:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949342.647405:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949342.647418:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949342.647421:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949342.647424:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949342.647428:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949342.647431:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949342.647436:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784949342.647439:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949342.647442:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000020:00000010:1.0:1784949342.647450:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000020:00000010:1.0:1784949342.647454:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000009adcfc93. 00000020:00000010:1.0:1784949342.647459:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000040:1.0:1784949342.647466:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:1.0:1784949342.647469:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949342.647471:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949342.647474:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949342.647480:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949342.647483:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949342.647484:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949342.647500:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949342.647511:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949342.647513:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:1.0:1784949342.647518:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 808 00000100:00000040:1.0:1784949342.647521:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949342.647524:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469565440 : -115544239986176 : ffff96e9c29eaa00) 00000100:00000040:1.0:1784949342.647530:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000925b9f50 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:248/0 lens 224/0 e 0 to 0 dl 1784949353 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949342.647542:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949342.647546:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:1.0:1784949342.647548:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949342.647551:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00100000:1.0:1784949342.647567:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000925b9f50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289691136:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949342.647573:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289691136 00000020:00000001:1.0:1784949342.647576:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949342.647580:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949342.647582:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949342.647585:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949342.647587:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949342.647591:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949342.647594:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949342.647596:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949342.647598:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949342.647600:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949342.647602:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949342.647606:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949342.647608:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:1.0:1784949342.647611:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000822137ec. 02000000:00000001:1.0:1784949342.647614:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949342.647617:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949342.647619:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949342.647622:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289691136 00010000:00000001:1.0:1784949342.647625:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:1.0:1784949342.647628:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000925b9f50 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:248/0 lens 224/224 e 0 to 0 dl 1784949353 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949342.647638:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:1.0:1784949342.647640:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949342.647643:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=59 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949342.647648:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1784949342.647650:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949342.647654:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949342.647657:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949342.647658:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949342.647661:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949342.647664:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949342.647668:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000100:00000200:1.0:1784949342.647671:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289691136, offset 224 00000400:00000200:1.0:1784949342.647677:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949342.647685:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949342.647692:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3925:3925:240:4294967295] 192.168.202.137@tcp LPNI seq info [2506:2506:8:4294967295] 00000400:00000200:1.0:1784949342.647703:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949342.647709:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949342.647713:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000cee99234. 00000800:00000200:1.0:1784949342.647719:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949342.647725:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949342.647728:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000cee99234 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949342.647744:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949342.647748:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949342.647751:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949342.647753:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949342.647755:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949342.647759:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000925b9f50 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:248/0 lens 224/224 e 0 to 0 dl 1784949353 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949342.647770:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000925b9f50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289691136:12345-192.168.202.137@tcp:400: Request processed in 227us (496us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949342.647780:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 808 00000100:00000040:1.0:1784949342.647783:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949342.647786:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949342.647789:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949342.647792:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000009adcfc93. 00000020:00000010:1.0:1784949342.647797:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949342.647803:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000020:00000040:1.0:1784949342.647808:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949342.647811:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949342.647874:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949342.647880:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000cee99234. 00000800:00000001:1.0:1784949342.647888:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949342.647898:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949342.647902:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949342.647936:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949342.647941:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949342.647955:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949342.647960:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949342.647970:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f400 00000400:00000200:2.0:1784949342.647972:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:2.0:1784949342.647981:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000100:00000001:2.0:1784949342.647989:0:337112:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949342.647993:0:337112:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:1.0:1784949342.648001:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9741 [1] + 224 00000800:00000001:1.0:1784949342.648012:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949342.648019:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949342.648022:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949342.648027:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949342.648042:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949342.648047:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949342.648059:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949342.648078:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949342.648085:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949342.648088:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.648206:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949342.648212:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949342.648219:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949342.648221:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949342.648223:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000084252b45. 00000400:00000200:3.0:1784949342.648231:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000400:00000010:3.0:1784949342.648234:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000100:00000001:3.0:1784949342.648238:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949342.648245:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949342.648258:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.648269:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949342.648271:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.648273:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949342.648276:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949342.648278:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949342.648279:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949342.648282:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949342.648289:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=59 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949342.648292:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=59 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949342.648296:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949342.648297:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949342.648299:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949342.648304:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949342.648321:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.648325:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949342.648327:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949342.648329:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949342.648331:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949342.648334:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.648342:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949342.648345:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949342.648346:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949342.648350:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949342.648358:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eac1e2f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289691136:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949342.648362:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.648365:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949342.648367:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eac1e2f5 x1871651289691136/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949342.648376:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949342.648378:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949342.648380:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 02000000:00000001:3.0:1784949342.648385:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949342.648386:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949342.648388:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949342.648390:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949342.648392:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000d104522b. 02000000:00000001:3.0:1784949342.648395:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949342.648397:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949342.648398:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eac1e2f5. 00000100:00000001:3.0:1784949342.648403:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949342.648404:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.648406:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949342.648412:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949342.648414:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949342.648416:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949342.648418:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949342.689020:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306282028, at 4306282029 00000100:00000040:2.0:1784949342.689029:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306282028, at 4306282029 00000100:00000040:1.0:1784949342.689040:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306282028, at 4306282029 00000100:00000010:3.0:1784949342.689043:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000010:2.0:1784949342.689050:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:1.0:1784949342.689060:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000040:0.0:1784949342.689064:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306282028, at 4306282029 00000100:00000010:0.0:1784949342.689084:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00010000:00000001:2.0:1784949343.475985:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949343.475993:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949343.476000:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949343.476002:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949343.476079:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949343.476083:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949343.476085:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949344.500079:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949344.500092:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949344.500145:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949344.500147:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949344.500230:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949344.500235:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949344.500238:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949345.523979:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949345.523987:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949345.523993:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949345.523995:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949345.524084:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949345.524105:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949345.524109:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949346.548011:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949346.548019:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949346.548025:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949346.548026:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949346.548139:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949346.548143:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949346.548146:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949347.571993:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949347.572003:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949347.572012:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949347.572014:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949347.572146:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949347.572150:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949347.572152:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949347.763992:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949347.763993:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949347.764002:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949347.764005:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949347.764008:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1784949347.764011:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000c6c2c60. 00000100:00000001:1.0:1784949347.764012:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949347.764016:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1784949347.764019:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949347.764020:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.764026:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949347.764028:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949347.764032:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949347.764037:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949347.764056:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000057c7ab4. 00000100:00000001:0.0:1784949347.764063:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949347.764086:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949347.764097:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949347.764099:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949347.764103:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c6c2c60] to pc [ptlrpcd_00_03+3] req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949347.764128:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949347.764130:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949347.764133:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949347.764135:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949347.764137:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949347.764140:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a3e309c2. 00000020:00000040:0.0:1784949347.764142:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949347.764143:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949347.764144:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949347.764146:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949347.764148:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003a8a98bc. 00000100:00000001:0.0:1784949347.764150:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949347.764155:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949347.764161:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949347.764162:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949347.764164:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e309c2] to pc [ptlrpcd_00_00+0] req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949347.764173:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949347.764178:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:3.0:1784949347.764180:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949347.764184:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949347.764186:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949347.764188:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11624) 00000100:00000001:3.0:1784949347.764189:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949347.764194:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949347.764197:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949347.764197:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949347.764200:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949347.764202:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949347.764207:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949347.764211:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949347.764214:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949347.764216:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1784949347.764219:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949347.764219:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949347.764220:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949347.764224:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.764228:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784949347.764228:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949347.764231:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949347.764233:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949347.764235:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949347.764237:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.764238:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949347.764240:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1784949347.764241:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000c6c2c60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289691264:0@lo:400: 00000100:00000001:3.0:1784949347.764246:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949347.764248:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949347.764252:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949347.764256:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.764261:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1784949347.764261:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949347.764264:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949347.764267:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:3.0:1784949347.764268:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000072ce5ef0. 02000000:00000001:1.0:1784949347.764269:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949347.764274:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949347.764274:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949347.764280:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a3e309c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289691392:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949347.764284:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000010:3.0:1784949347.764284:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000084252b45. 02000000:00000001:1.0:1784949347.764287:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949347.764289:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949347.764292:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:3.0:1784949347.764292:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004651e212. 02000000:00000010:1.0:1784949347.764296:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 02000000:00000001:1.0:1784949347.764299:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949347.764301:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289691264, portal 25 00000400:00000010:1.0:1784949347.764303:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000014bfbbb1. 00000100:00000001:3.0:1784949347.764304:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949347.764307:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469566336 : -115544239985280 : ffff96e9c29ead80) 00000400:00000010:1.0:1784949347.764310:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000100:00000040:3.0:1784949347.764314:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949347.764316:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289691392, portal 10 00000100:00000001:1.0:1784949347.764321:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949347.764323:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469567232 : -115544239984384 : ffff96e9c29eb100) 00000100:00000001:3.0:1784949347.764331:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949347.764335:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949347.764336:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949347.764341:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000100:00000200:3.0:1784949347.764346:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289691264, offset 0 00000400:00000200:3.0:1784949347.764358:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949347.764360:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949347.764364:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949347.764366:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000100:00000200:1.0:1784949347.764369:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289691392, offset 0 00000400:00000200:3.0:1784949347.764376:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784949347.764377:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949347.764398:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f480 00000400:00000200:1.0:1784949347.764405:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949347.764409:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8d49 [2] + 0 00000400:00000200:1.0:1784949347.764414:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000010:3.0:1784949347.764414:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000047084e78. 00000400:00000200:1.0:1784949347.764422:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949347.764423:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d4d2d632 00000400:00000200:1.0:1784949347.764426:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949347.764429:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949347.764432:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3926:3926:240:4294967295] 192.168.202.137@tcp LPNI seq info [2507:2507:8:4294967295] 00000400:00000200:3.0:1784949347.764432:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949347.764440:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949347.764445:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949347.764450:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949347.764453:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4d2d632 00000800:00000010:1.0:1784949347.764455:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b9b32818. 00000400:00000010:3.0:1784949347.764456:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d4d2d632. 00000100:00000001:3.0:1784949347.764463:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:1.0:1784949347.764466:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:3.0:1784949347.764467:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949347.764473:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000bc66e638 x1871651289691264 msgsize 224 00000800:00000200:1.0:1784949347.764480:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:3.0:1784949347.764481:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000200:1.0:1784949347.764486:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000b9b32818 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:3.0:1784949347.764487:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949347.764500:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949347.764505:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949347.764509:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000100:00000040:1.0:1784949347.764510:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2256 usecs (3 jiffies) 00000400:00000010:3.0:1784949347.764511:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:1.0:1784949347.764517:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306287107 00000100:00000001:3.0:1784949347.764518:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949347.764524:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949347.764525:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000001:1.0:1784949347.764534:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949347.764536:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949347.764540:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949347.764543:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306287107 00000100:00000001:0.0:1784949347.764547:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000010:1.0:1784949347.764549:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00100000:0.0:1784949347.764552:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289691264 02000000:00000001:0.0:1784949347.764556:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:1.0:1784949347.764557:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949347.764557:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949347.764561:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949347.764563:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949347.764565:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.764566:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306287107 00000100:00000001:3.0:1784949347.764570:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949347.764572:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949347.764573:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000031148565. 00000100:00000001:3.0:1784949347.764574:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306287146 00000100:00100000:0.0:1784949347.764580:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289691264 00000100:00000001:3.0:1784949347.764582:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306287107 00000020:00000001:0.0:1784949347.764583:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949347.764587:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949347.764588:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949347.764590:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306287146 00000020:00000001:0.0:1784949347.764591:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949347.764595:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306287107 00000020:00000040:0.0:1784949347.764596:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949347.764598:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306287107 00000020:00000001:0.0:1784949347.764602:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:1.0:1784949347.764603:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000300e92c9. 00000100:00000001:3.0:1784949347.764605:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306287146 00000020:00000001:0.0:1784949347.764607:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949347.764609:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784949347.764614:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306287107 00000100:00000001:0.0:1784949347.764615:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949347.764616:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949347.764622:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:1.0:1784949347.764622:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949347.764623:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306287146 00000100:00000001:0.0:1784949347.764628:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949347.764631:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000001:1.0:1784949347.764631:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949347.764632:0:337121:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306287107 00000100:00000001:1.0:1784949347.764639:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949347.764640:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949347.764641:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004565e281. 00000100:00000001:3.0:1784949347.764647:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949347.764649:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000001:3.0:1784949347.764652:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949347.764654:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000027aee7d. 00000100:00000001:3.0:1784949347.764658:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949347.764663:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949347.764667:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949347.764669:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949347.764671:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949347.764678:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949347.764685:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949347.764686:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949347.764690:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949347.764695:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b9b32818. 00000100:00000001:0.0:1784949347.764699:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000001:1.0:1784949347.764699:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949347.764700:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949347.764703:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:1.0:1784949347.764707:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:0.0:1784949347.764713:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 833 00000400:00000200:2.0:1784949347.764714:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949347.764715:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949347.764716:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949347.764718:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949347.764720:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949347.764722:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000100:00000001:0.0:1784949347.764724:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165112912 : -115539544438704 : ffff96eada7f0850) 00000100:00000001:3.0:1784949347.764724:0:342757:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949347.764726:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:1.0:1784949347.764731:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949347.764732:0:342757:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949347.764734:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784949347.764736:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000bc66e638 x1871651289691264/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:253/0 lens 224/0 e 0 to 0 dl 1784949358 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949347.764743:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949347.764749:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f500 00000100:00000001:0.0:1784949347.764751:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949347.764756:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:0.0:1784949347.764761:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949347.764762:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 182256 00000020:00000010:0.0:1784949347.764768:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000001:2.0:1784949347.764768:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949347.764771:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949347.764775:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949347.764776:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000bc66e638 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289691264:12345-0@lo:400: 00000800:00000001:1.0:1784949347.764781:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949347.764786:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289691264 00000400:00000200:1.0:1784949347.764788:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949347.764792:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:1.0:1784949347.764798:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784949347.764799:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:2.0:1784949347.764799:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949347.764803:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949347.764805:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1784949347.764807:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949347.764809:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:0.0:1784949347.764810:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000200:1.0:1784949347.764812:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:0.0:1784949347.764814:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949347.764817:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784949347.764819:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000029a32fda. 00000020:00000001:0.0:1784949347.764820:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949347.764822:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:1.0:1784949347.764824:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000029a32fda x1871651289691392 msgsize 224 00000020:00000001:0.0:1784949347.764826:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949347.764830:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:1.0:1784949347.764831:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949347.764835:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949347.764838:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949347.764842:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003bbd92b2. 02000000:00000001:0.0:1784949347.764844:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.764845:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949347.764846:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949347.764849:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949347.764852:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289691264 00000800:00000001:1.0:1784949347.764852:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:0.0:1784949347.764857:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:1.0:1784949347.764858:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:0.0:1784949347.764862:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000bc66e638 x1871651289691264/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:253/0 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949347.764869:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949347.764871:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949347.764874:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=69 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949347.764876:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784949347.764878:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1784949347.764880:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289691392 00000100:00000040:0.0:1784949347.764881:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 02000000:00000001:3.0:1784949347.764884:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949347.764885:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949347.764887:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949347.764889:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:3.0:1784949347.764890:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949347.764891:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949347.764896:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949347.764896:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949347.764901:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00100000:3.0:1784949347.764902:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289691392 00000400:00000010:0.0:1784949347.764904:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000020:00000001:3.0:1784949347.764931:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:0.0:1784949347.764932:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289691264, offset 224 00000020:00000040:3.0:1784949347.764933:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949347.764937:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949347.764940:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:0.0:1784949347.764941:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:3.0:1784949347.764945:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949347.764949:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784949347.764953:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949347.764955:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949347.764959:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dff9d289. 00000400:00000200:0.0:1784949347.764961:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f480 00000100:00000001:3.0:1784949347.764964:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949347.764967:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dff9d289. 00000400:00000200:0.0:1784949347.764970:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9755 [1] + 224 00000020:00000010:3.0:1784949347.764974:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f047fe3. 00000400:00000200:0.0:1784949347.764976:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949347.764980:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000010:3.0:1784949347.764981:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000007a8fede7. 00000100:00000001:0.0:1784949347.764983:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1784949347.764987:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000200:0.0:1784949347.764988:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949347.764996:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000040:0.0:1784949347.764999:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949347.765000:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949347.765003:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949347.765006:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949347.765014:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949347.765017:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949347.765018:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949347.765021:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949347.765028:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:0.0:1784949347.765030:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:3.0:1784949347.765032:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949347.765033:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949347.765036:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949347.765040:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949347.765044:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:3.0:1784949347.765044:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949347.765047:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1784949347.765055:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949347.765058:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949347.765058:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000001:0.0:1784949347.765059:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949347.765064:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:1.0:1784949347.765065:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:0.0:1784949347.765084:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bc66e638 x1871651289691264/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:253/0 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949347.765084:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1784949347.765085:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 809 00000400:00000010:1.0:1784949347.765089:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000084252b45. 00000100:00000001:2.0:1784949347.765090:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:3.0:1784949347.765092:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949347.765095:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:1.0:1784949347.765097:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004651e212 00000100:00000001:3.0:1784949347.765098:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637809152 : -115544071742464 : ffff96e9cca5dc00) 00000100:00000001:2.0:1784949347.765099:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949347.765103:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004651e212. 00000100:00000001:2.0:1784949347.765106:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.765109:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1784949347.765110:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000029a32fda x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:253/0 lens 224/0 e 0 to 0 dl 1784949358 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949347.765117:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949347.765120:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000bc66e638 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289691264:12345-0@lo:400: Request processed in 363us (641us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949347.765139:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00100000:0.0:1784949347.765142:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 833 00000020:00000010:3.0:1784949347.765144:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dff9d289. 00000100:00000200:1.0:1784949347.765145:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949347.765147:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949347.765151:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949347.765153:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949347.765157:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949347.765158:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dff9d289. 00000020:00000010:0.0:1784949347.765163:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00000001:1.0:1784949347.765167:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784949347.765169:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000027aee7d. 00000100:00100000:3.0:1784949347.765170:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000029a32fda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289691392:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949347.765171:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949347.765173:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004565e281. 00000100:00000001:1.0:1784949347.765176:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949347.765180:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000040:0.0:1784949347.765181:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000200:3.0:1784949347.765181:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289691392 00000100:00000001:1.0:1784949347.765183:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:3.0:1784949347.765186:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949347.765187:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949347.765189:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000020:00000001:3.0:1784949347.765190:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000001:1.0:1784949347.765191:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949347.765192:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000003bbd92b2. 00000020:00000001:3.0:1784949347.765193:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949347.765197:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=69 v=5 (5 5 5 5) 02000000:00000001:0.0:1784949347.765198:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000001:3.0:1784949347.765198:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949347.765202:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949347.765203:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00001000:1.0:1784949347.765204:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=69 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949347.765209:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000001:3.0:1784949347.765209:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949347.765211:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784949347.765214:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949347.765215:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949347.765216:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009b268f62. 00000020:00000001:3.0:1784949347.765221:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00010000:00000001:1.0:1784949347.765222:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949347.765224:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000400:00000010:0.0:1784949347.765225:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008e8d6490. 00010000:00000001:1.0:1784949347.765225:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949347.765227:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.765231:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949347.765232:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949347.765237:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000040:1.0:1784949347.765238:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949347.765240:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:3.0:1784949347.765245:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000020b8637f. 00000100:00000001:1.0:1784949347.765247:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949347.765249:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.765250:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949347.765252:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.765253:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949347.765256:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949347.765258:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:3.0:1784949347.765260:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289691392 00010000:00000001:3.0:1784949347.765265:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949347.765270:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000029a32fda x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:253/0 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949347.765272:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000c6c2c60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289691264:0@lo:400: 00000100:00000001:1.0:1784949347.765282:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949347.765285:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949347.765286:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:3.0:1784949347.765290:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949347.765291:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000c6c2c60 x1871651289691264/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:3.0:1784949347.765294:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=64 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949347.765301:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1784949347.765305:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:3.0:1784949347.765306:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949347.765309:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1784949347.765310:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000010:1.0:1784949347.765312:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000072ce5ef0. 02000000:00000001:3.0:1784949347.765313:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949347.765316:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949347.765318:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1784949347.765319:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1784949347.765321:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949347.765325:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000040:3.0:1784949347.765325:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949347.765330:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000431f37a5. 00000020:00000001:1.0:1784949347.765331:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000200:3.0:1784949347.765336:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289691392, offset 224 02000000:00000010:1.0:1784949347.765338:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000057c7ab4. 00000400:00000200:3.0:1784949347.765339:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949347.765344:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949347.765346:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949347.765347:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000c6c2c60. 00000400:00000200:3.0:1784949347.765351:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949347.765353:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949347.765356:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949347.765358:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3927:3927:240:4294967295] 192.168.202.137@tcp LPNI seq info [2508:2508:8:4294967295] 00000100:00000001:1.0:1784949347.765359:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949347.765365:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949347.765368:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949347.765373:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949347.765376:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:1.0:1784949347.765377:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949347.765385:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949347.765388:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949347.765393:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949347.765396:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949347.765399:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949347.765418:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949347.765421:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949347.765423:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949347.765424:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949347.765426:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949347.765429:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000029a32fda x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:253/0 lens 224/224 e 0 to 0 dl 1784949358 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949347.765436:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000029a32fda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289691392:12345-192.168.202.137@tcp:400: Request processed in 297us (611us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949347.765441:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 809 00000100:00000040:3.0:1784949347.765443:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949347.765445:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949347.765446:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949347.765448:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000007a8fede7. 00000020:00000010:3.0:1784949347.765451:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949347.765453:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f047fe3. 00000020:00000040:3.0:1784949347.765456:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949347.765458:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949347.765477:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949347.765480:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:1.0:1784949347.765484:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949347.765487:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949347.765490:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000431f37a5 00000400:00000010:1.0:1784949347.765491:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000431f37a5. 00000100:00000001:1.0:1784949347.765493:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949347.765495:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949347.765498:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949347.765501:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949347.765503:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949347.765504:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949347.765507:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949347.765513:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f500 00000400:00000200:1.0:1784949347.765517:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9759 [1] + 224 00000800:00000001:1.0:1784949347.765521:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949347.765524:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949347.765526:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949347.765528:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949347.765531:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949347.765534:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949347.765541:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949347.765555:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949347.765558:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949347.765560:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949347.765585:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949347.765590:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1784949347.765594:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1784949347.765596:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949347.765598:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000014bfbbb1. 00000400:00000200:2.0:1784949347.765604:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000db7d3f50 00000400:00000010:2.0:1784949347.765605:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000db7d3f50. 00000100:00000001:2.0:1784949347.765607:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949347.765612:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949347.765619:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949347.765625:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949347.765626:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949347.765627:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949347.765629:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949347.765630:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949347.765631:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949347.765633:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949347.765636:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=64 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949347.765639:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=64 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949347.765640:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949347.765642:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949347.765643:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1784949347.765645:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949347.765650:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949347.765652:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1784949347.765653:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:2.0:1784949347.765654:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1784949347.765655:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949347.765657:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949347.765662:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949347.765663:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949347.765664:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949347.765666:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949347.765679:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a3e309c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289691392:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949347.765683:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949347.765686:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949347.765688:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a3e309c2 x1871651289691392/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949347.765695:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949347.765698:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949347.765699:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:2.0:1784949347.765703:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1784949347.765705:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949347.765706:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:2.0:1784949347.765709:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949347.765711:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003a8a98bc. 02000000:00000001:2.0:1784949347.765714:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949347.765715:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949347.765716:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a3e309c2. 00000100:00000001:2.0:1784949347.765720:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949347.765721:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949347.765722:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949347.765726:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949347.765728:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949347.765730:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949347.765732:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949347.806977:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306287146, at 4306287147 00000100:00000010:1.0:1784949347.806992:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000040:2.0:1784949347.806996:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306287146, at 4306287147 00000100:00000040:3.0:1784949347.806998:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306287146, at 4306287147 00000100:00000040:0.0:1784949347.807000:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306287146, at 4306287147 00000100:00000010:2.0:1784949347.807006:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000031148565. 00000100:00000010:3.0:1784949347.807010:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000300e92c9. 00000100:00000010:0.0:1784949347.807013:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00010000:00000001:2.0:1784949348.596045:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949348.596055:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949348.596108:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949348.596127:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949348.596210:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949348.596217:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949348.596221:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949348.788101:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949348.788124:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949348.788128:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949348.788137:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949348.788143:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949348.788149:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949348.788154:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949349.620081:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949349.620091:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949349.620100:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949349.620102:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949349.620189:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949349.620196:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949349.620200:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949349.812196:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949349.812213:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949349.812215:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949349.812223:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949349.812226:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949349.812231:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949349.812234:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949350.644121:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949350.644128:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949350.644133:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949350.644134:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949350.644197:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949350.644201:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949350.644204:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949350.835998:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949350.836013:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949350.836015:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949350.836022:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949350.836025:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949350.836030:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949350.836032:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949351.667985:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949351.667992:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949351.667998:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949351.667999:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949351.668063:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949351.668068:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949351.668071:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949351.860161:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949351.860179:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949351.860182:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949351.860189:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949351.860205:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949351.860210:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949351.860218:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949352.692005:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949352.692013:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949352.692019:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949352.692022:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949352.692024:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949352.692043:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949352.692047:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949352.692048:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949352.692050:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949352.692052:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949352.692121:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949352.692129:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949352.692134:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1784949352.884123:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949352.884123:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949352.884139:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949352.884146:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:0.0:1784949352.884150:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a3e309c2. 00000100:00000001:1.0:1784949352.884152:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949352.884163:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784949352.884164:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949352.884172:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949352.884174:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949352.884182:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1784949352.884184:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949352.884193:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949352.884204:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949352.884219:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000635133da. 00000100:00000001:0.0:1784949352.884229:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949352.884242:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949352.884282:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949352.884284:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949352.884291:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e309c2] to pc [ptlrpcd_00_01+1] req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949352.884321:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949352.884329:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949352.884335:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949352.884339:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949352.884343:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949352.884347:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784949352.884348:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000c6c2c60. 00000020:00000040:0.0:1784949352.884356:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949352.884356:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949352.884362:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949352.884364:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784949352.884366:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949352.884371:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949352.884376:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000034aacccc. 00000100:00000040:2.0:1784949352.884376:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949352.884386:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949352.884393:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1784949352.884403:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949352.884404:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.884405:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949352.884412:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949352.884422:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949352.884425:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1784949352.884428:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949352.884429:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949352.884441:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949352.884445:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c6c2c60] to pc [ptlrpcd_00_02+2] req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949352.884450:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a3e309c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289691520:0@lo:400: 00000100:00000001:2.0:1784949352.884464:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949352.884473:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949352.884478:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949352.884487:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1784949352.884496:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005cc3cd5d. 02000000:00000001:2.0:1784949352.884499:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949352.884508:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000080bb5714. 00000100:00000001:0.0:1784949352.884509:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949352.884519:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11629) 00000400:00000010:2.0:1784949352.884521:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000957a03d0. 00000100:00000200:2.0:1784949352.884533:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289691520, portal 25 00000100:00000001:2.0:1784949352.884537:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949352.884539:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469567232 : -115544239984384 : ffff96e9c29eb100) 00000100:00000040:2.0:1784949352.884548:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949352.884551:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1784949352.884558:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:2.0:1784949352.884564:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949352.884566:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949352.884570:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949352.884573:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:3.0:1784949352.884578:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:2.0:1784949352.884580:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000100:00000001:1.0:1784949352.884581:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949352.884583:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949352.884588:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289691520, offset 0 00000100:00000001:3.0:1784949352.884595:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949352.884599:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1784949352.884605:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949352.884605:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1784949352.884619:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949352.884633:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949352.884635:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949352.884639:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1784949352.884644:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000001:3.0:1784949352.884645:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949352.884667:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949352.884679:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000000c6c2c60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289691648:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949352.884685:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949352.884689:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1784949352.884694:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f580 02000000:00000001:3.0:1784949352.884695:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949352.884706:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949352.884730:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000bf393d23. 00000400:00000200:2.0:1784949352.884731:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8d65 [2] + 0 02000000:00000001:3.0:1784949352.884740:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949352.884743:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002f32d921. 00000400:00000010:3.0:1784949352.884749:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000400:00000200:2.0:1784949352.884754:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000004764d328 00000400:00000200:2.0:1784949352.884766:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:3.0:1784949352.884769:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000400:00000200:2.0:1784949352.884775:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000200:3.0:1784949352.884779:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289691648, portal 10 00000400:00000200:2.0:1784949352.884784:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004764d328 00000100:00000001:3.0:1784949352.884785:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:2.0:1784949352.884792:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000004764d328. 00000100:00000001:3.0:1784949352.884793:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469566336 : -115544239985280 : ffff96e9c29ead80) 00000100:00000001:2.0:1784949352.884803:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000040:3.0:1784949352.884809:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949352.884810:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949352.884819:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000ace3646f x1871651289691520 msgsize 224 00000100:00100000:2.0:1784949352.884839:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:3.0:1784949352.884841:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949352.884843:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:3.0:1784949352.884851:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949352.884855:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003252ab93. 00000100:00000200:3.0:1784949352.884859:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289691648, offset 0 00000400:00000200:3.0:1784949352.884865:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949352.884870:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949352.884880:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949352.884888:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949352.884893:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949352.884897:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949352.884901:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949352.884940:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3928:3928:240:4294967295] 192.168.202.137@tcp LPNI seq info [2509:2509:8:4294967295] 00000400:00000200:3.0:1784949352.884957:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949352.884964:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949352.884970:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000800:00000010:3.0:1784949352.884971:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000400:00000010:2.0:1784949352.884977:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000100:00000001:0.0:1784949352.884985:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949352.884985:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784949352.884989:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:0.0:1784949352.884995:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289691520 00000100:00000200:2.0:1784949352.884996:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949352.885006:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000800:00000200:3.0:1784949352.885009:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784949352.885014:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:3.0:1784949352.885021:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949352.885054:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885054:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949352.885063:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949352.885085:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949352.885089:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949352.885099:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289691520 00000100:00000040:3.0:1784949352.885104:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2256 usecs (3 jiffies) 00000020:00000001:0.0:1784949352.885111:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949352.885117:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306292228 00000020:00000040:0.0:1784949352.885121:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:3.0:1784949352.885131:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000020:00000001:0.0:1784949352.885149:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949352.885152:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885158:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:0.0:1784949352.885160:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:3.0:1784949352.885165:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949352.885167:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949352.885172:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949352.885177:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306292267 00000100:00000001:3.0:1784949352.885179:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306292228 00000020:00000001:0.0:1784949352.885185:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:3.0:1784949352.885189:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:2.0:1784949352.885191:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306292228 00000100:00000001:0.0:1784949352.885198:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949352.885200:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949352.885203:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306292267 00000020:00000010:0.0:1784949352.885211:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:3.0:1784949352.885212:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306292228 00000100:00000001:2.0:1784949352.885214:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306292228 00000100:00000001:0.0:1784949352.885226:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1784949352.885226:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 00000100:00000001:2.0:1784949352.885227:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306292267 00000100:00000001:3.0:1784949352.885235:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000020:00000010:0.0:1784949352.885242:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:3.0:1784949352.885244:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306292228 00000100:00000001:2.0:1784949352.885248:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306292228 00000100:00000010:3.0:1784949352.885254:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:2.0:1784949352.885257:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306292267 00000100:00000001:3.0:1784949352.885261:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000020:00000010:0.0:1784949352.885263:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004565e281. 00000100:00000001:3.0:1784949352.885271:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949352.885275:0:337120:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306292228 00000100:00000001:3.0:1784949352.885277:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949352.885278:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000001:2.0:1784949352.885286:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949352.885288:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949352.885293:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000027aee7d. 00000100:00000001:2.0:1784949352.885297:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949352.885299:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949352.885303:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949352.885306:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885312:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1784949352.885313:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000100:00000040:0.0:1784949352.885319:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949352.885321:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1784949352.885324:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949352.885327:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1784949352.885330:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949352.885333:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949352.885339:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:1.0:1784949352.885340:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949352.885341:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:3.0:1784949352.885345:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949352.885348:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885349:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949352.885355:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003252ab93 00000100:00000001:2.0:1784949352.885359:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949352.885361:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003252ab93. 00000800:00000001:3.0:1784949352.885362:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949352.885369:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949352.885372:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949352.885372:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:3.0:1784949352.885375:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949352.885384:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949352.885398:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949352.885399:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:1.0:1784949352.885420:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1784949352.885426:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f600 00000100:00000040:1.0:1784949352.885432:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949352.885433:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949352.885434:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949352.885443:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949352.885444:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:3.0:1784949352.885446:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 182480 00000100:00100000:2.0:1784949352.885458:0:342757:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 834 00000800:00000001:3.0:1784949352.885467:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:2.0:1784949352.885471:0:342757:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949352.885475:0:342756:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949352.885479:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885479:0:342757:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165126224 : -115539544425392 : ffff96eada7f3c50) 00000800:00000001:3.0:1784949352.885481:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949352.885483:0:342756:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949352.885491:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784949352.885493:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949352.885496:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ace3646f x1871651289691520/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:258/0 lens 224/0 e 0 to 0 dl 1784949363 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949352.885509:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949352.885521:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949352.885525:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000020:00000010:2.0:1784949352.885529:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000031e3d331. 00000100:00000200:3.0:1784949352.885531:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:2.0:1784949352.885538:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1784949352.885541:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000e14e991b. 00000020:00000010:2.0:1784949352.885544:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000031e3d331. 00000100:00000040:3.0:1784949352.885551:0:337114:0:(events.c:337:request_in_callback()) incoming req@00000000e14e991b x1871651289691648 msgsize 224 00000100:00100000:2.0:1784949352.885559:0:342757:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ace3646f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289691520:12345-0@lo:400: 00000100:00100000:3.0:1784949352.885573:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:2.0:1784949352.885575:0:342757:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289691520 00000020:00000001:2.0:1784949352.885582:0:342757:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949352.885587:0:342757:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949352.885590:0:342757:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949352.885592:0:342757:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949352.885595:0:342757:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949352.885596:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:2.0:1784949352.885600:0:342757:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949352.885604:0:342757:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949352.885607:0:342757:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000800:00000001:3.0:1784949352.885610:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949352.885611:0:342757:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949352.885618:0:342757:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949352.885620:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949352.885622:0:342757:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949352.885629:0:342757:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949352.885635:0:342757:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949352.885640:0:342757:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000068fddc2b. 02000000:00000001:2.0:1784949352.885644:0:342757:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885648:0:342757:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949352.885651:0:342757:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949352.885655:0:342757:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289691520 00010000:00000001:2.0:1784949352.885659:0:342757:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949352.885665:0:342757:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ace3646f x1871651289691520/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:258/0 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949352.885675:0:342757:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949352.885677:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00010000:00000001:2.0:1784949352.885679:0:342757:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949352.885688:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289691648 00000100:00001000:2.0:1784949352.885688:0:342757:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=74 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949352.885698:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949352.885699:0:342757:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949352.885706:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000040:2.0:1784949352.885706:0:342757:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949352.885715:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885716:0:342757:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784949352.885723:0:342757:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949352.885726:0:342757:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949352.885727:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.885730:0:342757:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949352.885738:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289691648 00000100:00000040:2.0:1784949352.885738:0:342757:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949352.885742:0:342757:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000020:00000001:0.0:1784949352.885746:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:2.0:1784949352.885748:0:342757:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289691520, offset 224 00000020:00000040:0.0:1784949352.885751:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:2.0:1784949352.885759:0:342757:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:0.0:1784949352.885761:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949352.885769:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:2.0:1784949352.885769:0:342757:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:0.0:1784949352.885780:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784949352.885782:0:342757:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f580 00000020:00000001:0.0:1784949352.885793:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:2.0:1784949352.885797:0:342757:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9771 [1] + 224 00000100:00000001:0.0:1784949352.885805:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:2.0:1784949352.885808:0:342757:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:0.0:1784949352.885814:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000400:00000200:2.0:1784949352.885833:0:342757:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949352.885845:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949352.885847:0:342757:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949352.885856:0:342757:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949352.885858:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000020:00000010:0.0:1784949352.885876:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001c9428b0. 00000100:00000040:2.0:1784949352.885876:0:342757:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949352.885890:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a7c5da82. 00000020:00000010:0.0:1784949352.885901:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000075e36e9f. 00000100:00000040:0.0:1784949352.885916:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949352.885922:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949352.885935:0:342757:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949352.885944:0:342757:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:2.0:1784949352.885947:0:342757:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000100:00000001:2.0:1784949352.885950:0:342757:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949352.885952:0:342757:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949352.885972:0:342757:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949352.885976:0:342757:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:2.0:1784949352.885979:0:342757:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949352.885981:0:342757:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949352.885984:0:342757:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949352.885988:0:342757:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ace3646f x1871651289691520/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:258/0 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949352.885995:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949352.886003:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949352.886012:0:342757:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ace3646f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289691520:12345-0@lo:400: Request processed in 485us (1185us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949352.886017:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00100000:2.0:1784949352.886065:0:342757:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 834 00000100:00000001:0.0:1784949352.886068:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000040:2.0:1784949352.886072:0:342757:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949352.886073:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949352.886075:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949352.886080:0:342757:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949352.886081:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949352.886086:0:342757:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000400:00000001:1.0:1784949352.886093:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:2.0:1784949352.886095:0:342757:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00000001:0.0:1784949352.886097:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:1.0:1784949352.886099:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949352.886106:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000080bb5714. 00000020:00000010:2.0:1784949352.886106:0:342757:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000027aee7d. 00000020:00000010:2.0:1784949352.886117:0:342757:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004565e281. 00000400:00000200:1.0:1784949352.886120:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000957a03d0 00000400:00000010:1.0:1784949352.886126:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000957a03d0. 00000100:00000001:0.0:1784949352.886127:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000040:2.0:1784949352.886130:0:342757:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:1.0:1784949352.886133:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1784949352.886136:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:2.0:1784949352.886141:0:342757:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:3.0:1784949352.886143:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000200:1.0:1784949352.886147:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784949352.886147:0:342757:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000068fddc2b. 00000100:00000001:3.0:1784949352.886151:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949352.886159:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 810 02000000:00000001:2.0:1784949352.886160:0:342757:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:2.0:1784949352.886167:0:342757:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949352.886176:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:3.0:1784949352.886176:0:342763:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949352.886181:0:342757:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000200:1.0:1784949352.886185:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949352.886190:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554949888 : -115544154601728 : ffff96e9c7b58700) 00000100:00000001:3.0:1784949352.886190:0:342763:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949352.886193:0:342757:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000002f32d921. 00000400:00000010:2.0:1784949352.886208:0:342757:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004764d328. 00000100:00000001:1.0:1784949352.886214:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1784949352.886217:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e14e991b x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:258/0 lens 224/0 e 0 to 0 dl 1784949363 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949352.886220:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949352.886228:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949352.886234:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949352.886239:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949352.886246:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949352.886253:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949352.886263:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=74 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949352.886273:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=74 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949352.886285:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949352.886287:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949352.886288:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949352.886297:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:0.0:1784949352.886300:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00010000:00000001:1.0:1784949352.886301:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949352.886307:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949352.886311:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1784949352.886315:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1784949352.886323:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:1.0:1784949352.886336:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949352.886345:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:0.0:1784949352.886347:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e14e991b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289691648:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949352.886348:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949352.886358:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949352.886367:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289691648 00000100:00100000:1.0:1784949352.886377:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a3e309c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289691520:0@lo:400: 00000020:00000001:0.0:1784949352.886380:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949352.886388:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949352.886390:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00000001:1.0:1784949352.886393:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1784949352.886397:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949352.886399:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a3e309c2 x1871651289691520/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949352.886410:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949352.886414:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784949352.886418:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 02000000:00000001:1.0:1784949352.886419:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949352.886425:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000005cc3cd5d. 00000020:00000001:0.0:1784949352.886430:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 02000000:00000001:1.0:1784949352.886432:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949352.886438:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:0.0:1784949352.886439:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1784949352.886441:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:1.0:1784949352.886450:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:0.0:1784949352.886451:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 02000000:00000010:1.0:1784949352.886454:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000635133da. 00000020:00000001:0.0:1784949352.886460:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 02000000:00000001:1.0:1784949352.886461:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1784949352.886469:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949352.886469:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949352.886473:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a3e309c2. 00000020:00000001:0.0:1784949352.886479:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949352.886481:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949352.886487:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949352.886490:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:1.0:1784949352.886491:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949352.886500:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949352.886502:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949352.886507:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949352.886510:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1784949352.886511:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003bbd92b2. 00000100:00000001:1.0:1784949352.886513:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949352.886523:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949352.886529:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949352.886534:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949352.886539:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289691648 00010000:00000001:0.0:1784949352.886544:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949352.886553:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e14e991b x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:258/0 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949352.886569:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949352.886573:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949352.886577:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=69 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949352.886584:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949352.886589:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949352.886595:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949352.886600:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949352.886603:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949352.886608:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949352.886614:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949352.886620:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a89b27e6. 00000100:00000200:0.0:1784949352.886627:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289691648, offset 224 00000400:00000200:0.0:1784949352.886637:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949352.886650:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949352.886659:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3929:3929:240:4294967295] 192.168.202.137@tcp LPNI seq info [2510:2510:8:4294967295] 00000400:00000200:0.0:1784949352.886677:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949352.886687:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949352.886693:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009ff40a64. 00000800:00000200:0.0:1784949352.886702:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949352.886712:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:0.0:1784949352.886717:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009ff40a64 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949352.886745:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949352.886752:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949352.886756:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949352.886759:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949352.886764:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949352.886774:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e14e991b x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:258/0 lens 224/224 e 0 to 0 dl 1784949363 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949352.886795:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e14e991b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289691648:12345-192.168.202.137@tcp:400: Request processed in 506us (1240us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949352.886810:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 810 00000100:00000040:0.0:1784949352.886816:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949352.886821:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949352.886825:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949352.886831:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a7c5da82. 00000020:00000010:0.0:1784949352.886840:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000075e36e9f. 00000020:00000010:0.0:1784949352.886847:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001c9428b0. 00000020:00000040:0.0:1784949352.886855:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949352.886860:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949352.886896:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949352.886902:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009ff40a64. 00000400:00000200:1.0:1784949352.886937:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949352.886945:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949352.886950:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a89b27e6 00000400:00000010:1.0:1784949352.886953:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a89b27e6. 00000100:00000001:1.0:1784949352.886958:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949352.886960:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949352.886967:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949352.886975:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949352.886979:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949352.886984:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949352.886990:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949352.887001:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f600 00000400:00000200:1.0:1784949352.887008:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9779 [1] + 224 00000800:00000001:1.0:1784949352.887015:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949352.887022:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949352.887053:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949352.887058:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949352.887062:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949352.887067:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949352.887078:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949352.887098:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949352.887105:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949352.887108:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949352.887207:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949352.887213:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949352.887220:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949352.887222:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949352.887224:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000400:00000200:3.0:1784949352.887234:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:3.0:1784949352.887236:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:3.0:1784949352.887240:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949352.887247:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949352.887261:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949352.887271:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949352.887273:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949352.887276:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949352.887278:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949352.887280:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949352.887282:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949352.887285:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949352.887291:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=69 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949352.887295:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=69 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949352.887298:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949352.887300:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949352.887302:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949352.887306:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949352.887315:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949352.887319:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949352.887320:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949352.887323:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949352.887325:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949352.887329:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949352.887337:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949352.887340:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949352.887342:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949352.887345:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949352.887353:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000000c6c2c60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289691648:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949352.887358:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949352.887360:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949352.887362:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000000c6c2c60 x1871651289691648/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949368 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949352.887370:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949352.887372:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949352.887374:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000bf393d23. 02000000:00000001:3.0:1784949352.887378:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949352.887380:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949352.887381:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949352.887383:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949352.887385:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000034aacccc. 02000000:00000001:3.0:1784949352.887388:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949352.887390:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949352.887391:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000c6c2c60. 00000100:00000001:3.0:1784949352.887396:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949352.887397:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949352.887399:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949352.887405:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949352.887406:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949352.887408:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949352.887411:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949352.927998:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306292267, at 4306292268 00000100:00000010:2.0:1784949352.928015:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00000100:00000040:1.0:1784949352.928074:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306292267, at 4306292268 00000100:00000040:0.0:1784949352.928077:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306292267, at 4306292268 00000100:00000040:3.0:1784949352.928085:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306292267, at 4306292268 00000100:00000010:1.0:1784949352.928089:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000010:0.0:1784949352.928098:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000010:3.0:1784949352.928102:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00010000:00000001:2.0:1784949353.716082:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949353.716092:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949353.716099:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949353.716101:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949353.716178:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949353.716185:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949353.716190:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949353.908061:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949353.908077:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949353.908080:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949353.908088:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949353.908092:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949353.908097:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949353.908100:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949354.740067:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949354.740077:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949354.740086:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949354.740087:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949354.740212:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949354.740219:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949354.740223:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949354.932049:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949354.932069:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949354.932073:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949354.932081:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949354.932085:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949354.932091:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949354.932095:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949355.764031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949355.764042:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949355.764052:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949355.764054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949355.764150:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949355.764156:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949355.764161:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949355.955987:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949355.956012:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949355.956014:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949355.956019:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949355.956022:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949355.956025:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949355.956028:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949356.788095:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949356.788104:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949356.788111:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949356.788112:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949356.788181:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949356.788186:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949356.788190:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949356.980043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949356.980055:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949356.980058:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949356.980063:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949356.980066:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949356.980069:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949356.980072:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949357.812082:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949357.812092:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949357.812100:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949357.812102:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949357.812196:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949357.812204:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949357.812208:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949358.004051:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949358.004067:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949358.004070:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949358.004073:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949358.004076:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949358.004082:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949358.004086:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949358.004089:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.004095:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949358.004096:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eac1e2f5. 00000020:00000040:0.0:1784949358.004107:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949358.004113:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949358.004118:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949358.004125:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949358.004135:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005a1c05a3. 00000100:00000001:0.0:1784949358.004145:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949358.004156:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949358.004171:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949358.004174:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949358.004181:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eac1e2f5] to pc [ptlrpcd_00_03+3] req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949358.004216:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949358.004219:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949358.004224:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949358.004227:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949358.004231:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949358.004234:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003bf8e8ad. 00000020:00000040:0.0:1784949358.004237:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949358.004239:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949358.004243:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949358.004245:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949358.004250:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949358.004253:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949358.004261:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949358.004266:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949358.004274:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:0.0:1784949358.004276:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1784949358.004280:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949358.004282:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.004294:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1784949358.004297:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf8e8ad] to pc [ptlrpcd_00_00+0] req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949358.004299:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949358.004310:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949358.004316:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949358.004316:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949358.004325:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1784949358.004327:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949358.004338:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949358.004345:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949358.004357:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11635) 00000100:00000001:1.0:1784949358.004360:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949358.004365:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949358.004370:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1784949358.004371:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949358.004375:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784949358.004378:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 02000000:00000001:1.0:1784949358.004382:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949358.004385:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949358.004392:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eac1e2f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289691776:0@lo:400: 00000100:00000001:3.0:1784949358.004397:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949358.004399:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949358.004403:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949358.004409:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949358.004410:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1784949358.004416:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949358.004417:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949358.004424:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1784949358.004426:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949358.004435:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 00000100:00000040:3.0:1784949358.004438:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949358.004442:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949358.004442:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949358.004453:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949358.004459:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000001:3.0:1784949358.004473:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:1.0:1784949358.004475:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000100:00000001:3.0:1784949358.004477:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949358.004486:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000200:1.0:1784949358.004490:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289691776, portal 25 02000000:00000001:3.0:1784949358.004491:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949358.004501:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1784949358.004502:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.004507:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469569920 : -115544239981696 : ffff96e9c29ebb80) 00000100:00100000:3.0:1784949358.004512:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289691904:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949358.004519:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1784949358.004521:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949358.004526:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949358.004533:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949358.004541:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949358.004544:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949358.004551:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000010:3.0:1784949358.004551:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000035b54a67. 02000000:00000001:3.0:1784949358.004560:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949358.004561:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000100:00000200:1.0:1784949358.004569:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289691776, offset 0 00000400:00000010:3.0:1784949358.004570:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000400:00000010:3.0:1784949358.004585:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000400:00000200:1.0:1784949358.004588:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000200:3.0:1784949358.004609:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289691904, portal 10 00000400:00000200:1.0:1784949358.004610:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949358.004620:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949358.004624:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469562752 : -115544239988864 : ffff96e9c29e9f80) 00000400:00000200:1.0:1784949358.004648:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f680 00000100:00000040:3.0:1784949358.004650:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949358.004669:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8d81 [2] + 0 00000100:00000001:3.0:1784949358.004672:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784949358.004680:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000002e5b8264. 00000100:00000040:3.0:1784949358.004686:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949358.004690:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d51a2e44 00000400:00000010:3.0:1784949358.004694:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f4f06fd5. 00000100:00000200:3.0:1784949358.004701:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289691904, offset 0 00000400:00000200:1.0:1784949358.004703:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949358.004715:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949358.004716:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949358.004725:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d51a2e44 00000400:00000010:1.0:1784949358.004730:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d51a2e44. 00000400:00000200:3.0:1784949358.004736:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949358.004740:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949358.004745:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1784949358.004746:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1784949358.004755:0:337121:0:(events.c:337:request_in_callback()) incoming req@000000006f58d1a4 x1871651289691776 msgsize 224 00000400:00000200:3.0:1784949358.004759:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949358.004771:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00100000:1.0:1784949358.004775:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784949358.004782:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000040:1.0:1784949358.004783:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949358.004793:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3930:3930:240:4294967295] 192.168.202.137@tcp LPNI seq info [2511:2511:8:4294967295] 00000100:00000001:1.0:1784949358.004807:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:3.0:1784949358.004813:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949358.004822:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949358.004829:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000400:00000200:1.0:1784949358.004830:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000400:00000010:1.0:1784949358.004838:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000100:00000001:1.0:1784949358.004846:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:3.0:1784949358.004850:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:1.0:1784949358.004858:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949358.004871:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000200:3.0:1784949358.004876:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:0.0:1784949358.004879:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289691776 00000100:00000001:1.0:1784949358.004881:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1784949358.004887:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 02000000:00000001:0.0:1784949358.004889:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:1.0:1784949358.004893:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949358.004896:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949358.004932:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949358.004940:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949358.004954:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289691776 00000100:00000001:1.0:1784949358.004955:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949358.004962:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.004964:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1784949358.004965:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949358.004974:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000040:3.0:1784949358.004974:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2257 usecs (3 jiffies) 00000100:00000001:1.0:1784949358.004977:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949358.005005:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949358.005007:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306297387 00000100:00000001:3.0:1784949358.005014:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306297348 00000020:00000040:0.0:1784949358.005016:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000010:1.0:1784949358.005018:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f595b698. 00000020:00000001:0.0:1784949358.005029:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949358.005046:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1784949358.005047:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949358.005060:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306297387 00000100:00000001:0.0:1784949358.005061:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949358.005062:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306297387 00000100:00000001:3.0:1784949358.005072:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306297348 00000100:00000010:1.0:1784949358.005074:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000300e92c9. 00000020:00000010:0.0:1784949358.005075:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000001:0.0:1784949358.005086:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949358.005087:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000020:00000010:0.0:1784949358.005093:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000100:00000001:3.0:1784949358.005099:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306297387 00000100:00000001:1.0:1784949358.005100:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306297387 00000020:00000010:0.0:1784949358.005106:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001c9428b0. 00000100:00000001:3.0:1784949358.005111:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306297348 00000100:00000010:1.0:1784949358.005115:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000020:00000010:0.0:1784949358.005117:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000a7c5da82. 00000100:00000001:1.0:1784949358.005124:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000010:0.0:1784949358.005127:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000075e36e9f. 00000100:00000001:1.0:1784949358.005133:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306297387 00000100:00000001:3.0:1784949358.005133:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306297387 00000100:00000040:0.0:1784949358.005142:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:3.0:1784949358.005146:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306297348 00000100:00000010:1.0:1784949358.005148:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:0.0:1784949358.005150:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949358.005157:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949358.005160:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949358.005162:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949358.005176:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306297387 00000100:00000001:1.0:1784949358.005177:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949358.005179:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949358.005186:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949358.005187:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949358.005189:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949358.005198:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000100:00000001:3.0:1784949358.005201:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949358.005202:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949358.005203:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949358.005211:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949358.005217:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.005219:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949358.005230:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949358.005230:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949358.005231:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949358.005240:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949358.005248:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f4f06fd5 00000100:00100000:0.0:1784949358.005256:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 835 00000400:00000010:2.0:1784949358.005257:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f4f06fd5. 00000100:00000040:0.0:1784949358.005267:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949358.005270:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949358.005273:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534165114960 : -115539544436656 : ffff96eada7f1050) 00000100:00000200:2.0:1784949358.005282:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949358.005285:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000006f58d1a4 x1871651289691776/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:264/0 lens 224/0 e 0 to 0 dl 1784949369 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949358.005287:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949358.005307:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949358.005317:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949358.005323:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949358.005347:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949358.005352:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949358.005354:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1784949358.005359:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000009888ee6. 00000100:00000040:2.0:1784949358.005367:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949358.005369:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f700 00000100:00000001:0.0:1784949358.005370:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949358.005383:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000009888ee6. 00000400:00000200:3.0:1784949358.005398:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 182704 00000100:00100000:0.0:1784949358.005400:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000006f58d1a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289691776:12345-0@lo:400: 00000800:00000001:3.0:1784949358.005421:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784949358.005422:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289691776 00000100:00000001:2.0:1784949358.005424:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949358.005432:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:2.0:1784949358.005432:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1784949358.005436:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949358.005441:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:3.0:1784949358.005444:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1784949358.005446:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949358.005452:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000200:3.0:1784949358.005458:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:0.0:1784949358.005459:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949358.005469:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949358.005474:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949358.005475:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:0.0:1784949358.005478:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000200:3.0:1784949358.005480:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:0.0:1784949358.005483:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949358.005490:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949358.005491:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fbc2d61e. 00000020:00000001:0.0:1784949358.005494:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000040:3.0:1784949358.005500:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000fbc2d61e x1871651289691904 msgsize 224 00000100:00000001:0.0:1784949358.005502:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949358.005510:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:3.0:1784949358.005517:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000010:0.0:1784949358.005520:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000bcfc6676. 02000000:00000001:0.0:1784949358.005526:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949358.005529:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949358.005532:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949358.005536:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289691776 00000100:00000001:3.0:1784949358.005539:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000001:0.0:1784949358.005540:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949358.005550:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000006f58d1a4 x1871651289691776/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:264/0 lens 224/224 e 0 to 0 dl 1784949369 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:3.0:1784949358.005553:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949358.005562:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784949358.005565:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949358.005570:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949358.005575:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=80 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949358.005581:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949358.005587:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949358.005590:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:2.0:1784949358.005594:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 02000000:00000001:0.0:1784949358.005595:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949358.005602:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949358.005603:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289691904 00000100:00000001:0.0:1784949358.005607:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1784949358.005609:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:0.0:1784949358.005617:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:2.0:1784949358.005618:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949358.005624:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949358.005626:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002e7a7cb2. 02000000:00000001:2.0:1784949358.005629:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949358.005635:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289691776, offset 224 00000100:00100000:2.0:1784949358.005636:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289691904 00000020:00000001:2.0:1784949358.005645:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949358.005648:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:0.0:1784949358.005650:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000020:00000001:2.0:1784949358.005655:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949358.005660:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:2.0:1784949358.005668:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000400:00000200:0.0:1784949358.005669:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000020:00000001:2.0:1784949358.005675:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949358.005683:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949358.005689:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000400:00000200:0.0:1784949358.005693:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f680 00000100:00000001:2.0:1784949358.005696:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949358.005708:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000020:00000010:2.0:1784949358.005721:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005cc3cd5d. 00000400:00000200:0.0:1784949358.005723:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x978d [1] + 224 00000020:00000010:2.0:1784949358.005729:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000006dae6aed. 00000020:00000010:2.0:1784949358.005739:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000400:00000200:0.0:1784949358.005742:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949358.005748:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:0.0:1784949358.005758:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949358.005758:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949358.005764:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949358.005770:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949358.005773:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949358.005777:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:0.0:1784949358.005783:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949358.005784:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949358.005790:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949358.005806:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949358.005818:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949358.005834:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949358.005835:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949358.005841:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:0.0:1784949358.005850:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e7a7cb2 00000100:00100000:2.0:1784949358.005852:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 811 00000400:00000010:0.0:1784949358.005857:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002e7a7cb2. 00000100:00000040:2.0:1784949358.005859:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949358.005866:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949358.005866:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554960640 : -115544154590976 : ffff96e9c7b5b100) 00000100:00000001:0.0:1784949358.005872:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000040:2.0:1784949358.005879:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000fbc2d61e x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:264/0 lens 224/0 e 0 to 0 dl 1784949369 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949358.005881:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949358.005894:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784949358.005935:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949358.005941:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.005945:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1784949358.005947:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:1.0:1784949358.005952:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949358.005962:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006f58d1a4 x1871651289691776/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:264/0 lens 224/224 e 0 to 0 dl 1784949369 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949358.005964:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784949358.005965:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000001:1.0:1784949358.005972:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:2.0:1784949358.005974:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000400:00000010:1.0:1784949358.005979:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000100:00000001:2.0:1784949358.006003:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949358.006013:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000400:00000200:1.0:1784949358.006031:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000100:00100000:0.0:1784949358.006036:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000006f58d1a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289691776:12345-0@lo:400: Request processed in 674us (1271us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949358.006038:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000fbc2d61e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289691904:12345-192.168.202.137@tcp:400: 00000400:00000010:1.0:1784949358.006040:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000100:00000001:1.0:1784949358.006050:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949358.006059:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289691904 00000100:00000200:1.0:1784949358.006062:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949358.006066:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:0.0:1784949358.006071:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 835 00000020:00000001:2.0:1784949358.006074:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949358.006081:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949358.006083:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:2.0:1784949358.006091:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949358.006093:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:2.0:1784949358.006098:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:0.0:1784949358.006101:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:1.0:1784949358.006107:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949358.006109:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000010:0.0:1784949358.006112:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000a7c5da82. 00000020:00000001:2.0:1784949358.006118:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949358.006124:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000075e36e9f. 00000020:00000001:2.0:1784949358.006130:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000010:0.0:1784949358.006135:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001c9428b0. 00000020:00000001:2.0:1784949358.006137:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949358.006144:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.006148:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:0.0:1784949358.006150:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000020:00000001:2.0:1784949358.006151:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949358.006153:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949358.006160:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949358.006162:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949358.006162:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949358.006168:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949358.006170:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000bcfc6676. 02000000:00000001:1.0:1784949358.006171:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000010:2.0:1784949358.006175:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d58187af. 00000100:00000001:1.0:1784949358.006177:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949358.006179:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:1.0:1784949358.006184:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949358.006187:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949358.006188:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1784949358.006191:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949358.006198:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949358.006203:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:1.0:1784949358.006204:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=80 v=5 (5 5 5 5) 00000020:00000001:2.0:1784949358.006205:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949358.006217:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289691904 00000400:00000010:0.0:1784949358.006218:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000055244775. 00000100:00001000:1.0:1784949358.006218:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=80 v=5 (1 1 1 1) 00010000:00000001:2.0:1784949358.006230:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949358.006232:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:0.0:1784949358.006234:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004e790d63. 00000100:00000001:1.0:1784949358.006237:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:2.0:1784949358.006241:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000fbc2d61e x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:264/0 lens 224/224 e 0 to 0 dl 1784949369 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949358.006247:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949358.006254:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949358.006264:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949358.006266:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949358.006272:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949358.006278:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:2.0:1784949358.006279:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=75 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949358.006295:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949358.006304:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949358.006310:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949358.006311:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000100:00000001:1.0:1784949358.006321:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1784949358.006322:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00000001:1.0:1784949358.006328:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949358.006329:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949358.006341:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949358.006344:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949358.006352:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949358.006361:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000967f0319. 00000100:00100000:1.0:1784949358.006368:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eac1e2f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289691776:0@lo:400: 00000100:00000200:2.0:1784949358.006371:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289691904, offset 224 00000100:00000001:1.0:1784949358.006378:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949358.006387:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1784949358.006387:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949358.006397:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eac1e2f5 x1871651289691776/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949358.006409:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949358.006420:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000400:00000200:2.0:1784949358.006428:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3931:3931:240:4294967295] 192.168.202.137@tcp LPNI seq info [2512:2512:8:4294967295] 02000000:00000001:1.0:1784949358.006429:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949358.006434:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:1.0:1784949358.006443:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949358.006449:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949358.006456:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000400:00000200:2.0:1784949358.006462:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:1.0:1784949358.006464:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949358.006470:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005a1c05a3. 02000000:00000001:1.0:1784949358.006477:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:2.0:1784949358.006478:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 02000000:00000001:1.0:1784949358.006482:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000010:2.0:1784949358.006487:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f15a9a04. 00000100:00000010:1.0:1784949358.006488:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eac1e2f5. 00000100:00000001:1.0:1784949358.006499:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949358.006504:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949358.006505:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949358.006509:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949358.006518:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:2.0:1784949358.006519:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:1.0:1784949358.006522:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949358.006529:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f15a9a04 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949358.006531:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949358.006537:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949358.006557:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949358.006565:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949358.006568:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949358.006571:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949358.006573:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949358.006577:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fbc2d61e x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:264/0 lens 224/224 e 0 to 0 dl 1784949369 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949358.006589:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000fbc2d61e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289691904:12345-192.168.202.137@tcp:400: Request processed in 624us (1087us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949358.006598:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 811 00000100:00000040:2.0:1784949358.006601:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949358.006603:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949358.006605:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949358.006608:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000006dae6aed. 00000020:00000010:2.0:1784949358.006613:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 00000020:00000010:2.0:1784949358.006617:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005cc3cd5d. 00000020:00000040:2.0:1784949358.006622:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949358.006625:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949358.006805:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949358.006813:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949358.006815:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f15a9a04. 00000400:00000200:0.0:1784949358.006825:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949358.006826:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949358.006834:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949358.006835:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784949358.006839:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949358.006843:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000967f0319 00000400:00000010:0.0:1784949358.006850:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000967f0319. 00000400:00000200:2.0:1784949358.006857:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949358.006858:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949358.006864:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949358.006876:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f700 00000400:00000200:2.0:1784949358.006885:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9795 [1] + 224 00000800:00000001:2.0:1784949358.006893:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949358.006947:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949358.006950:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949358.006955:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949358.006960:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949358.006964:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949358.006976:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949358.007025:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949358.007032:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949358.007048:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949358.007119:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949358.007125:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949358.007132:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949358.007134:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949358.007137:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000400:00000200:3.0:1784949358.007146:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:3.0:1784949358.007148:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:3.0:1784949358.007152:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949358.007160:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949358.007174:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949358.007185:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949358.007186:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949358.007189:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949358.007192:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949358.007194:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949358.007196:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949358.007198:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949358.007204:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=75 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949358.007208:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=75 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949358.007211:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949358.007212:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949358.007215:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949358.007219:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949358.007228:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949358.007231:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949358.007233:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949358.007236:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949358.007237:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949358.007240:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949358.007249:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949358.007251:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949358.007253:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949358.007257:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949358.007265:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289691904:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949358.007270:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949358.007272:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949358.007275:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003bf8e8ad x1871651289691904/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949358.007283:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949358.007285:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949358.007288:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000035b54a67. 02000000:00000001:3.0:1784949358.007294:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949358.007295:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949358.007297:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949358.007300:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949358.007302:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:3.0:1784949358.007305:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949358.007306:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949358.007308:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003bf8e8ad. 00000100:00000001:3.0:1784949358.007313:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949358.007315:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949358.007317:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949358.007322:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949358.007324:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949358.007326:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949358.007329:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949358.048050:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306297387, at 4306297388 00000100:00000040:2.0:1784949358.048060:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306297387, at 4306297388 00000100:00000010:3.0:1784949358.048068:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:2.0:1784949358.048076:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000040:1.0:1784949358.048092:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306297387, at 4306297388 00000100:00000040:0.0:1784949358.048097:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306297387, at 4306297388 00000100:00000010:1.0:1784949358.048108:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000300e92c9. 00000100:00000010:0.0:1784949358.048112:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f595b698. 00010000:00000001:2.0:1784949358.836043:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949358.836053:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949358.836061:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949358.836063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949358.836156:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949358.836162:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949358.836168:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949359.028211:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949359.028227:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949359.028229:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949359.028235:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949359.028239:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949359.028243:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949359.028262:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949359.860056:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949359.860065:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949359.860075:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949359.860077:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949359.860206:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949359.860213:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949359.860217:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949360.052078:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949360.052096:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949360.052098:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949360.052106:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949360.052110:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949360.052114:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949360.052118:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949360.884026:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949360.884033:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949360.884040:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949360.884041:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949360.884109:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949360.884113:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949360.884116:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1784949361.076046:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1784949361.076062:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949361.076065:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949361.076072:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949361.076077:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949361.076081:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949361.076084:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949361.908078:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949361.908089:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949361.908097:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949361.908099:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949361.908187:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949361.908193:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949361.908198:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949362.100058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949362.100072:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949362.100074:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949362.100079:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949362.100082:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949362.100084:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949362.100087:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949362.932060:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949362.932069:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949362.932077:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949362.932082:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949362.932085:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949362.932088:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949362.932092:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949362.932095:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949362.932097:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949362.932100:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949362.932187:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949362.932194:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949362.932198:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949363.124012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949363.124027:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949363.124030:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949363.124036:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949363.124039:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949363.124042:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949363.124045:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949363.124047:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949363.124055:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949363.124065:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f37d62c3. 00000020:00000040:2.0:1784949363.124073:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949363.124078:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949363.124081:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949363.124087:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949363.124095:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003cb28a6d. 00000100:00000001:2.0:1784949363.124102:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949363.124111:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949363.124123:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949363.124125:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949363.124130:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f37d62c3] to pc [ptlrpcd_00_01+1] req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949363.124161:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949363.124163:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949363.124167:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949363.124170:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949363.124173:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949363.124175:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006aa8881d. 00000020:00000040:2.0:1784949363.124177:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949363.124179:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949363.124180:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949363.124182:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949363.124185:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000284ae7c9. 00000100:00000001:2.0:1784949363.124188:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949363.124193:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949363.124194:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949363.124202:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1784949363.124204:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949363.124206:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949363.124207:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949363.124214:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa8881d] to pc [ptlrpcd_00_02+2] req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949363.124215:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949363.124220:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949363.124222:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949363.124227:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949363.124231:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949363.124231:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949363.124236:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949363.124245:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949363.124254:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949363.124256:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949363.124261:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949363.124261:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11640) 00000100:00000001:3.0:1784949363.124266:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949363.124267:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949363.124272:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784949363.124272:0:337122:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00000001:3.0:1784949363.124279:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949363.124280:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949363.124287:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f37d62c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289692032:0@lo:400: 00000100:00000001:3.0:1784949363.124288:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949363.124290:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949363.124295:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1784949363.124295:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949363.124297:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949363.124301:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1784949363.124303:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1784949363.124305:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1784949363.124306:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949363.124309:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784949363.124311:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1784949363.124317:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001c9428b0. 00000100:00000040:3.0:1784949363.124318:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949363.124323:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949363.124334:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:0.0:1784949363.124335:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bc6c6508. 00000100:00000001:3.0:1784949363.124337:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949363.124343:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949363.124345:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784949363.124346:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002e7a7cb2. 02000000:00000001:3.0:1784949363.124349:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949363.124353:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006aa8881d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289692160:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949363.124357:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000200:0.0:1784949363.124358:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289692032, portal 25 02000000:00000001:3.0:1784949363.124361:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949363.124362:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949363.124367:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469571968 : -115544239979648 : ffff96e9c29ec380) 02000000:00000001:3.0:1784949363.124367:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949363.124373:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1784949363.124375:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784949363.124381:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000035b54a67. 00000100:00000001:0.0:1784949363.124385:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1784949363.124388:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949363.124390:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949363.124395:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000100:00000200:0.0:1784949363.124399:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289692032, offset 0 00000400:00000010:3.0:1784949363.124399:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000400:00000200:0.0:1784949363.124412:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000010:3.0:1784949363.124412:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000200:3.0:1784949363.124422:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289692160, portal 10 00000100:00000001:3.0:1784949363.124426:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949363.124428:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469577344 : -115544239974272 : ffff96e9c29ed880) 00000400:00000200:0.0:1784949363.124432:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000040:3.0:1784949363.124444:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949363.124453:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f780 00000100:00000001:3.0:1784949363.124465:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1784949363.124475:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8da5 [2] + 0 00000100:00000040:3.0:1784949363.124476:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949363.124481:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004ed69935. 00000400:00000010:0.0:1784949363.124489:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009ef01e88. 00000400:00000200:0.0:1784949363.124502:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000d1181415 00000100:00000200:3.0:1784949363.124510:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289692160, offset 0 00000400:00000200:0.0:1784949363.124513:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949363.124520:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949363.124525:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:3.0:1784949363.124535:0:337122:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949363.124540:0:337122:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949363.124545:0:337122:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:3.0:1784949363.124549:0:337122:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1784949363.124551:0:337122:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:3.0:1784949363.124554:0:337122:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3932:3932:240:4294967295] 192.168.202.137@tcp LPNI seq info [2513:2513:8:4294967295] 00000400:00000200:3.0:1784949363.124565:0:337122:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949363.124571:0:337122:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949363.124572:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d1181415 00000800:00000010:3.0:1784949363.124577:0:337122:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003547cd5f. 00000400:00000010:0.0:1784949363.124578:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000d1181415. 00000100:00000001:0.0:1784949363.124588:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000800:00000200:3.0:1784949363.124591:0:337122:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000200:0.0:1784949363.124592:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949363.124599:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000e78df062 x1871651289692032 msgsize 224 00000800:00000200:3.0:1784949363.124610:0:337122:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:0.0:1784949363.124611:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949363.124618:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:3.0:1784949363.124619:0:337122:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003547cd5f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949363.124631:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949363.124640:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000100:00000001:3.0:1784949363.124641:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949363.124642:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000100:00000001:0.0:1784949363.124648:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1784949363.124648:0:337122:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2260 usecs (3 jiffies) 00000100:00000001:3.0:1784949363.124654:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306302467 00000100:00000200:0.0:1784949363.124656:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1784949363.124664:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000071d4fc18. 00000100:00000001:0.0:1784949363.124669:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949363.124674:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949363.124675:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949363.124685:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306302467 00000100:00000001:0.0:1784949363.124686:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949363.124694:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:3.0:1784949363.124695:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003d48e6d9. 00000100:00000001:0.0:1784949363.124701:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949363.124701:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:3.0:1784949363.124705:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306302467 00000100:00000001:0.0:1784949363.124707:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306302506 00000100:00000010:3.0:1784949363.124710:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000024fe3fef. 00000100:00000001:3.0:1784949363.124716:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949363.124717:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306302467 00000100:00000001:2.0:1784949363.124721:0:342757:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949363.124722:0:337122:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306302467 00000100:00000001:0.0:1784949363.124725:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306302506 00000100:00100000:2.0:1784949363.124727:0:342757:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289692032 00000100:00000010:3.0:1784949363.124729:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000077cd14ae. 02000000:00000001:2.0:1784949363.124734:0:342757:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949363.124737:0:337122:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:0.0:1784949363.124738:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306302467 00000100:00000001:2.0:1784949363.124739:0:342757:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949363.124743:0:342757:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949363.124749:0:342757:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949363.124750:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306302506 00000100:00000001:3.0:1784949363.124750:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1784949363.124757:0:342757:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289692032 00000100:00000001:3.0:1784949363.124759:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949363.124762:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306302467 00000020:00000001:2.0:1784949363.124763:0:342757:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:3.0:1784949363.124766:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949363.124767:0:342757:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:0.0:1784949363.124770:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306302506 00000800:00000200:1.0:1784949363.124770:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:2.0:1784949363.124774:0:342757:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949363.124775:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1784949363.124778:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003547cd5f. 00000020:00000040:2.0:1784949363.124780:0:342757:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784949363.124785:0:342757:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784949363.124787:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949363.124788:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306302467 00000020:00000001:2.0:1784949363.124789:0:342757:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949363.124800:0:342757:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949363.124801:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1784949363.124802:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000010:2.0:1784949363.124807:0:342757:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000008aed2f1. 00000100:00000001:0.0:1784949363.124810:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949363.124812:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004ed69935 00000100:00000001:2.0:1784949363.124813:0:342757:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949363.124814:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949363.124818:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004ed69935. 00000020:00000010:2.0:1784949363.124821:0:342757:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000008aed2f1. 00000800:00000001:3.0:1784949363.124822:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949363.124823:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949363.124825:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:2.0:1784949363.124831:0:342757:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005cc3cd5d. 00000100:00000200:1.0:1784949363.124834:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949363.124840:0:342757:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cca15944. 00000800:00000001:3.0:1784949363.124841:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784949363.124850:0:342757:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000022b9b2f5. 00000800:00000001:3.0:1784949363.124851:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949363.124861:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949363.124863:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949363.124865:0:342757:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949363.124870:0:342757:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000040:1.0:1784949363.124871:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949363.124874:0:342757:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949363.124880:0:342757:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949363.124882:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:2.0:1784949363.124893:0:342757:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949363.124898:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949363.124902:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1784949363.124961:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f800 00000100:00000001:2.0:1784949363.124965:0:342757:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949363.124973:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:3.0:1784949363.124975:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 182928 00000100:00000001:0.0:1784949363.124978:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:2.0:1784949363.124980:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949363.124984:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:3.0:1784949363.124987:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:0.0:1784949363.124988:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 836 00000100:00000040:0.0:1784949363.124994:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949363.124996:0:342757:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949363.124998:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782156880 : -115538927394736 : ffff96eaff466050) 00000800:00000001:3.0:1784949363.124999:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949363.125000:0:342757:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949363.125008:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e78df062 x1871651289692032/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:269/0 lens 224/0 e 0 to 0 dl 1784949374 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949363.125008:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949363.125018:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949363.125029:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949363.125030:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000020:00000010:0.0:1784949363.125034:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000200:3.0:1784949363.125035:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:0.0:1784949363.125042:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1784949363.125044:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000774d655b. 00000020:00000010:0.0:1784949363.125046:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00000040:3.0:1784949363.125051:0:337113:0:(events.c:337:request_in_callback()) incoming req@00000000774d655b x1871651289692160 msgsize 224 00000100:00100000:0.0:1784949363.125057:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e78df062 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289692032:12345-0@lo:400: 00000100:00100000:3.0:1784949363.125064:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000200:0.0:1784949363.125066:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289692032 00000020:00000001:0.0:1784949363.125071:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949363.125076:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949363.125078:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949363.125080:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949363.125081:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:3.0:1784949363.125083:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000001:0.0:1784949363.125084:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949363.125089:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949363.125092:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949363.125094:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949363.125097:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1784949363.125098:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949363.125100:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949363.125105:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000001:3.0:1784949363.125108:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949363.125109:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949363.125115:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008b893a44. 02000000:00000001:0.0:1784949363.125119:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949363.125121:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:0.0:1784949363.125124:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949363.125125:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289692160 00000020:00000001:0.0:1784949363.125128:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949363.125130:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00010000:00000040:0.0:1784949363.125134:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289692032 00000100:00000001:1.0:1784949363.125134:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949363.125138:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949363.125140:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:1.0:1784949363.125143:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000200:0.0:1784949363.125147:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e78df062 x1871651289692032/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:269/0 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949363.125149:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289692160 00000020:00000001:1.0:1784949363.125154:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949363.125157:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949363.125161:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949363.125166:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00010000:00000001:0.0:1784949363.125168:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000001:1.0:1784949363.125171:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:0.0:1784949363.125172:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949363.125176:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00001000:0.0:1784949363.125179:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=85 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949363.125181:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949363.125187:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00000100:00000001:0.0:1784949363.125188:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949363.125194:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:1.0:1784949363.125194:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949363.125201:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000100:00000001:0.0:1784949363.125202:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949363.125208:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949363.125210:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949363.125211:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000005ae2686. 00000100:00000001:0.0:1784949363.125212:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949363.125218:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000010:1.0:1784949363.125218:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000013f48f48. 00000400:00000010:0.0:1784949363.125225:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000020:00000010:1.0:1784949363.125226:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000200:0.0:1784949363.125229:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289692032, offset 224 00000100:00000040:1.0:1784949363.125235:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000200:0.0:1784949363.125238:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949363.125243:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949363.125247:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000400:00000200:0.0:1784949363.125250:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949363.125254:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949363.125260:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f780 00000100:00000001:1.0:1784949363.125264:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000400:00000200:0.0:1784949363.125267:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x97a9 [1] + 224 00000100:00000001:1.0:1784949363.125270:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949363.125274:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949363.125275:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949363.125283:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949363.125286:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949363.125289:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949363.125298:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949363.125299:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949363.125318:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949363.125321:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949363.125325:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1784949363.125330:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 812 00000100:00000040:1.0:1784949363.125333:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000400:00000200:0.0:1784949363.125334:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000100:00000001:1.0:1784949363.125339:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554954368 : -115544154597248 : ffff96e9c7b59880) 00000400:00000010:0.0:1784949363.125340:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000100:00000001:0.0:1784949363.125347:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000040:1.0:1784949363.125349:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000774d655b x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:269/0 lens 224/0 e 0 to 0 dl 1784949374 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949363.125351:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:0.0:1784949363.125359:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949363.125361:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949363.125364:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:2.0:1784949363.125366:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949363.125368:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000010:1.0:1784949363.125370:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b6bd2181. 00010000:00000001:0.0:1784949363.125372:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949363.125376:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1784949363.125376:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1784949363.125377:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1784949363.125382:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1784949363.125385:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000010:1.0:1784949363.125386:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b6bd2181. 00000400:00000010:2.0:1784949363.125387:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000bc6c6508. 00000100:00000040:0.0:1784949363.125393:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e78df062 x1871651289692032/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:269/0 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949363.125397:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002e7a7cb2 00000100:00100000:1.0:1784949363.125400:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000774d655b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289692160:12345-192.168.202.137@tcp:400: 00000400:00000010:2.0:1784949363.125402:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002e7a7cb2. 00000100:00000001:2.0:1784949363.125410:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949363.125415:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289692160 00000100:00000200:2.0:1784949363.125419:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949363.125421:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00100000:0.0:1784949363.125426:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e78df062 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289692032:12345-0@lo:400: Request processed in 391us (819us total) trans 0 rc 0/0 00000020:00000001:1.0:1784949363.125427:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949363.125432:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949363.125439:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949363.125444:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:1.0:1784949363.125452:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00100000:0.0:1784949363.125453:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 836 00000020:00000001:1.0:1784949363.125456:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949363.125456:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949363.125463:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:0.0:1784949363.125465:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:1.0:1784949363.125468:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949363.125472:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949363.125475:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000020:00000001:1.0:1784949363.125477:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949363.125477:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949363.125483:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:2.0:1784949363.125484:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949363.125485:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949363.125490:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:2.0:1784949363.125491:0:337119:0:(client.c:1580:after_reply()) Process entered 00000020:00000010:0.0:1784949363.125493:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cca15944. 02000000:00000010:1.0:1784949363.125497:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000078a163a8. 02000000:00000001:2.0:1784949363.125497:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949363.125501:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000010:0.0:1784949363.125502:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000022b9b2f5. 02000000:00000001:1.0:1784949363.125502:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949363.125504:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949363.125510:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949363.125512:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949363.125513:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005cc3cd5d. 00000020:00000001:1.0:1784949363.125514:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949363.125518:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=85 v=5 (5 5 5 5) 00010000:00000040:1.0:1784949363.125520:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289692160 00000020:00000040:0.0:1784949363.125525:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00010000:00000001:1.0:1784949363.125526:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00001000:2.0:1784949363.125526:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=85 v=5 (1 1 1 1) 02000000:00000001:0.0:1784949363.125534:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000200:1.0:1784949363.125535:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000774d655b x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:269/0 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949363.125535:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 02000000:00000010:0.0:1784949363.125539:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008b893a44. 00000100:00000001:2.0:1784949363.125541:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949363.125550:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000001:2.0:1784949363.125550:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949363.125552:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:0.0:1784949363.125557:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1784949363.125557:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949363.125557:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949363.125567:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=80 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949363.125568:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949363.125569:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:1.0:1784949363.125578:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1784949363.125580:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000313d28ec. 00000100:00000040:2.0:1784949363.125580:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949363.125585:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000400:00000010:0.0:1784949363.125590:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000099bab33a. 00000100:00000001:1.0:1784949363.125593:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949363.125600:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949363.125605:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949363.125606:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949363.125611:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949363.125611:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949363.125616:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949363.125621:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:2.0:1784949363.125624:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949363.125627:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000100:00000200:1.0:1784949363.125634:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289692160, offset 224 00000100:00100000:2.0:1784949363.125639:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f37d62c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289692032:0@lo:400: 00000400:00000200:1.0:1784949363.125648:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949363.125650:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949363.125654:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949363.125658:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f37d62c3 x1871651289692032/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949363.125661:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949363.125674:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3933:3933:240:4294967295] 192.168.202.137@tcp LPNI seq info [2514:2514:8:4294967295] 00000100:00000001:2.0:1784949363.125675:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1784949363.125679:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1784949363.125683:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001c9428b0. 02000000:00000001:2.0:1784949363.125690:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:1.0:1784949363.125695:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:2.0:1784949363.125695:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949363.125700:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000800:00000200:1.0:1784949363.125706:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784949363.125707:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949363.125711:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003cb28a6d. 00000800:00000010:1.0:1784949363.125713:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007a3f8609. 02000000:00000001:2.0:1784949363.125716:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949363.125719:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:1.0:1784949363.125724:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000010:2.0:1784949363.125724:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f37d62c3. 00000100:00000001:2.0:1784949363.125732:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000800:00000200:1.0:1784949363.125737:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:2.0:1784949363.125737:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949363.125742:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949363.125744:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000007a3f8609 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949363.125748:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949363.125751:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949363.125754:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949363.125756:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949363.125761:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949363.125766:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949363.125768:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949363.125771:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949363.125773:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:1.0:1784949363.125777:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000774d655b x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:269/0 lens 224/224 e 0 to 0 dl 1784949374 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949363.125785:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000774d655b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337122:x1871651289692160:12345-192.168.202.137@tcp:400: Request processed in 420us (733us total) trans 0 rc 0/0 00000100:00100000:1.0:1784949363.125792:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 812 00000100:00000040:1.0:1784949363.125794:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949363.125796:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949363.125798:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949363.125800:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000013f48f48. 00000020:00000010:1.0:1784949363.125803:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949363.125806:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000005ae2686. 00000020:00000040:1.0:1784949363.125809:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949363.125811:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949363.126000:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949363.126011:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949363.126015:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949363.126019:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949363.126028:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000800:00000200:3.0:1784949363.126049:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1784949363.126056:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f800 00000800:00000010:3.0:1784949363.126057:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007a3f8609. 00000400:00000200:1.0:1784949363.126065:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x97b1 [1] + 224 00000400:00000200:3.0:1784949363.126067:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:1.0:1784949363.126073:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949363.126080:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949363.126083:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949363.126085:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ecb574ad 00000400:00000200:1.0:1784949363.126087:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:3.0:1784949363.126090:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ecb574ad. 00000400:00000200:1.0:1784949363.126096:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949363.126097:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949363.126101:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949363.126102:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000200:1.0:1784949363.126108:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949363.126117:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949363.126128:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949363.126133:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949363.126136:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949363.126158:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949363.126163:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949363.126167:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949363.126170:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949363.126172:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000400:00000200:3.0:1784949363.126177:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:3.0:1784949363.126180:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:3.0:1784949363.126183:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949363.126189:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949363.126200:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949363.126208:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949363.126209:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949363.126211:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949363.126213:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949363.126214:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949363.126216:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949363.126218:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949363.126223:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=80 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949363.126226:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=80 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949363.126228:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949363.126229:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949363.126231:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949363.126234:0:337122:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949363.126240:0:337122:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949363.126258:0:337122:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949363.126260:0:337122:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949363.126263:0:337122:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949363.126264:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949363.126268:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949363.126275:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949363.126278:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949363.126280:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949363.126284:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949363.126292:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006aa8881d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:337122:1871651289692160:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949363.126297:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949363.126299:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949363.126301:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006aa8881d x1871651289692160/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949363.126308:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949363.126310:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949363.126312:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000035b54a67. 02000000:00000001:3.0:1784949363.126318:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949363.126320:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949363.126321:0:337122:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949363.126324:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949363.126326:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000284ae7c9. 02000000:00000001:3.0:1784949363.126329:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949363.126330:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949363.126332:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006aa8881d. 00000100:00000001:3.0:1784949363.126335:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949363.126336:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949363.126338:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949363.126343:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949363.126345:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949363.126347:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949363.126349:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949363.167008:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306302506, at 4306302507 00000100:00000010:2.0:1784949363.167022:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000024fe3fef. 00000100:00000040:0.0:1784949363.167026:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306302506, at 4306302507 00000100:00000040:1.0:1784949363.167037:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306302506, at 4306302507 00000100:00000010:0.0:1784949363.167040:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000071d4fc18. 00000100:00000010:1.0:1784949363.167054:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003d48e6d9. 00000100:00000040:3.0:1784949363.167057:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306302506, at 4306302507 00000100:00000010:3.0:1784949363.167071:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000077cd14ae. 00010000:00000001:2.0:1784949363.956052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949363.956060:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949363.956066:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949363.956067:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949363.956161:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949363.956168:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949363.956173:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949364.148036:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949364.148049:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949364.148051:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949364.148056:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949364.148059:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949364.148062:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949364.148065:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949364.980038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949364.980047:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949364.980055:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949364.980058:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949364.980144:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949364.980151:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949364.980156:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949365.172012:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949365.172025:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949365.172027:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949365.172032:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949365.172035:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949365.172039:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949365.172041:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1784949365.684096:0:344055:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 1.2.3.4:988 00000400:00000100:2.0:1784949365.684170:0:344055:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 1.2.3.4@tcp at host 1.2.3.4:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949365.684185:0:344055:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 1.2.3.4@tcp: down 00000400:00000200:2.0:1784949365.684194:0:344055:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1784949365.684202:0:344055:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->1.2.3.4@tcp 00000800:00000010:2.0:1784949365.684208:0:344055:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000adf08f9b. 00000400:00000200:2.0:1784949365.684218:0:344055:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:2.0:1784949365.684226:0:344055:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949365.684234:0:344055:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11638 health :900 00000400:00000200:2.0:1784949365.684242:0:344055:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949365.684247:0:344055:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000400:00000200:2.0:1784949365.684253:0:344055:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:1.0:1784949365.684324:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949365.684336:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:1.0:1784949365.684357:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949365.684359:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949365.684363:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f071c401 00000400:00000010:1.0:1784949365.684366:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000f071c401. 00000400:00000200:1.0:1784949365.684371:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949365.684375:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000010:1.0:1784949365.684379:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000059e5b358. 00000400:00000200:1.0:1784949365.684385:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-110 00000400:00000200:1.0:1784949365.684387:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -110 00000400:00000200:1.0:1784949365.684391:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00010000:00000001:2.0:1784949366.004051:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949366.004061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949366.004069:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949366.004071:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949366.004187:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949366.004195:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949366.004203:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949366.196058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949366.196075:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949366.196079:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949366.196085:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949366.196089:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949366.196093:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949366.196096:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949366.387994:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001f2bdc7c. 00000400:00000010:1.0:1784949366.388011:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000325e9e7. 00000400:00000010:1.0:1784949366.388018:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000ca3adacb. 00000400:00000010:1.0:1784949366.388021:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008c288862. 00000400:00000010:1.0:1784949366.388023:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f12dd04d. 00000400:00000010:1.0:1784949366.388031:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009e0071c7. 00000400:00000010:1.0:1784949366.388035:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000041ec6752. 00000400:00000010:1.0:1784949366.388037:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f65f2fb7. 00000400:00000010:1.0:1784949366.388039:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003b463cff. 00000400:00000010:1.0:1784949366.388041:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000091a157d3. 00000400:00000010:1.0:1784949366.388043:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000006d2c4b24. 00000400:00000010:1.0:1784949366.388045:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000171e706a. 00000400:00000010:1.0:1784949366.388046:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000339be500. 00000400:00000010:1.0:1784949366.388048:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a8bf092e. 00000400:00000010:1.0:1784949366.388050:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000cef8b3cd. 00000400:00000010:1.0:1784949366.388051:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009ec02d9a. 00000400:00000010:1.0:1784949366.388053:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e3882e96. 00000400:00000010:1.0:1784949366.388054:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000014336a14. 00000400:00000010:1.0:1784949366.388056:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000a3f431e3. 00000400:00000010:1.0:1784949366.388058:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000007e2f3d0. 00000400:00000010:1.0:1784949366.388059:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000f9fb66e4. 00000400:00000010:1.0:1784949366.388061:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000d3823a3e. 00000400:00000010:1.0:1784949366.388063:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000e3180272. 00000400:00000010:1.0:1784949366.388064:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009f8441cb. 00000400:00000010:1.0:1784949366.388066:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000851e368. 00000400:00000010:1.0:1784949366.388068:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000022dbcdce. 00000400:00000010:1.0:1784949366.388069:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000003859b1c5. 00000400:00000010:1.0:1784949366.388071:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000009234a5fd. 00000400:00000010:1.0:1784949366.388072:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000451e3fe4. 00000400:00000010:1.0:1784949366.388074:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000acbf6818. 00000400:00000010:1.0:1784949366.388076:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000008c245eeb. 00000400:00000010:1.0:1784949366.388077:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000004fd5390b. 00000400:00000010:1.0:1784949366.388079:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001ff78f22. 00000400:00000010:1.0:1784949366.388080:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000b4c93ba0. 00000400:00000010:1.0:1784949366.388082:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000038468d93. 00000400:00000010:1.0:1784949366.388084:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000074c2f2ff. 00000400:00000010:1.0:1784949366.388085:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 00000000da72cb2d. 00000400:00000010:1.0:1784949366.388090:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000001fcbecf4. 00000400:00000010:1.0:1784949366.388093:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 0000000056670828. 00000400:00000010:1.0:1784949366.388095:0:337116:0:(lib-lnet.h:450:lnet_rspt_free()) free 'rspt': 64 at 000000000a5d96d2. 00000400:00000200:1.0:1784949366.388102:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:1.0:1784949366.388109:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000e9215ad7. 00000400:00000010:1.0:1784949366.388118:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:1.0:1784949366.388124:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000400:00000010:1.0:1784949366.388131:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000000a5d96d2. 00000400:00000200:1.0:1784949366.388135:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949366.388150:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949366.388157:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949366.388161:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3934:3934:241:4294967295] 192.168.202.137@tcp LPNI seq info [2515:2515:8:4294967295] 00000400:00000200:1.0:1784949366.388168:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:1.0:1784949366.388175:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:1.0:1784949366.388180:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949366.388185:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:1.0:1784949366.388191:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949366.388199:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949366.388204:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009e7fe55b type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949366.388224:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11640 health :900 00000800:00000200:3.0:1784949366.388358:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949366.388365:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:3.0:1784949366.388371:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949366.388377:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949366.388383:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949366.388386:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949366.388398:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949366.388410:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949366.388413:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949366.388419:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949366.388426:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949366.388438:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949366.388447:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949366.388456:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949366.388460:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3935:3935:241:4294967295] 192.168.202.137@tcp LPNI seq info [2516:2516:8:4294967295] 00000400:00000200:3.0:1784949366.388468:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949366.388474:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949366.388478:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000800:00000200:3.0:1784949366.388484:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949366.388491:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949366.388495:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 3, nob 144 niov 1 nkiov 1 00000800:00000001:1.0:1784949366.388546:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949366.388604:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949366.388609:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000800:00000001:2.0:1784949366.388612:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949366.388614:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949366.388623:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:2.0:1784949366.388625:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949366.388631:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949366.388635:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949366.388638:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949366.388646:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949366.388662:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x97c5 00000800:00000001:2.0:1784949366.388667:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949366.388676:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949366.388678:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949366.388681:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949366.388687:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000400:00000010:2.0:1784949366.388689:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000400:00000200:2.0:1784949366.388692:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949366.388695:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000e9215ad7. 00000400:00000010:2.0:1784949366.388698:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000800:00000001:2.0:1784949366.388704:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949366.388707:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949367.028053:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949367.028061:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949367.028070:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949367.028072:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949367.028155:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949367.028159:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949367.028162:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949367.220007:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949367.220014:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784949367.220029:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949342/real 1784949342] req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949367 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1784949367.220050:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949367.220052:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949367.220055:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000877cac20. 00000400:00000200:1.0:1784949367.220063:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:1.0:1784949367.220066:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:1.0:1784949367.220071:0:337118:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949367.220076:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949367 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949367.220086:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949367 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949367.220093:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949367.220095:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949367.220097:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949367.220099:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949367.220103:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949367 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949367.220109:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949367.220118:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949367.220120:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949367.220123:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949367.220126:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949367.220129:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949367.220134:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949367 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949367.220142:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949367.220145:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949367.220147:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949367.220148:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949367.220151:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784949367.220153:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784949367.220156:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784949367.220184:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00000001:1.0:1784949367.220187:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784949367.220192:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949367 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949367.220202:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289691008:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784949367.220211:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949367.220215:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949367.220218:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000aedb1960 x1871651289691008/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949367 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949367.220225:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949367.220229:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949367.220233:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000004cb3d562. 02000000:00000001:1.0:1784949367.220240:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949367.220243:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949367.220246:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:1.0:1784949367.220249:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949367.220252:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000b1427c95. 02000000:00000001:1.0:1784949367.220257:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949367.220259:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949367.220261:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000aedb1960. 00000100:00000001:1.0:1784949367.220267:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949367.220269:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949367.220270:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1784949367.220271:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949367.220277:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949367.220281:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949367.220284:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949367.220286:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949367.220290:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1784949367.220297:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004614f9cb. 00000020:00000040:0.0:1784949367.220304:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949367.220310:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949367.220313:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949367.220320:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949367.220328:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002a753d9d. 00000100:00000001:0.0:1784949367.220335:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949367.220342:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949367.220353:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949367.220356:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949367.220361:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004614f9cb] to pc [ptlrpcd_00_03+3] req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949367.220398:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949367.220401:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949367.220405:0:89766:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949367.220409:0:89766:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949367.220411:0:89766:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949367.220416:0:89766:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949367.220421:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11614 00000100:00000001:2.0:1784949367.220423:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949367.220429:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949367.220434:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784949367.220435:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1784949367.220440:0:89766:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11613 00000100:00000040:2.0:1784949367.220446:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949367.220452:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949367.220459:0:89766:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000100:00000001:3.0:1784949367.220459:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.220466:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1784949367.220470:0:89766:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:2.0:1784949367.220471:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949367.220478:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 02000000:00000001:2.0:1784949367.220479:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1784949367.220485:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949367.220485:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949367.220488:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 02000000:00000001:2.0:1784949367.220492:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949367.220493:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949367.220500:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000200:3.0:1784949367.220500:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00100000:2.0:1784949367.220501:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289692288:0@lo:400: 00000020:00000001:0.0:1784949367.220510:0:89766:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949367.220512:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1784949367.220513:0:89766:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000400:00000010:3.0:1784949367.220514:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000052e1e01a. 00000020:00000001:0.0:1784949367.220519:0:89766:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1784949367.220520:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:3.0:1784949367.220523:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 02000000:00000001:2.0:1784949367.220526:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949367.220527:0:89766:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 02000000:00000001:2.0:1784949367.220532:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1784949367.220533:0:89766:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000400:00000010:3.0:1784949367.220534:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d6b48e85. 02000000:00000010:2.0:1784949367.220540:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000005cc3cd5d. 00000020:00000001:0.0:1784949367.220542:0:89766:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:2.0:1784949367.220547:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.220549:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000040:0.0:1784949367.220554:0:89766:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00000400:00000010:2.0:1784949367.220557:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000054d5ba64. 00000100:00000001:0.0:1784949367.220559:0:89766:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949367.220565:0:89766:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000200:3.0:1784949367.220570:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:0.0:1784949367.220571:0:89766:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:2.0:1784949367.220571:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000507ec8db. 00000020:00000040:0.0:1784949367.220581:0:89766:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00000200:2.0:1784949367.220583:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289692288, portal 25 00000400:00000200:3.0:1784949367.220586:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784949367.220588:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1784949367.220590:0:89766:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 00000100:00000001:2.0:1784949367.220593:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469554688 : -115544239996928 : ffff96e9c29e8000) 00000100:00080000:0.0:1784949367.220600:0:89766:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000200:3.0:1784949367.220601:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000040:2.0:1784949367.220603:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949367.220609:0:89766:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.220616:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784949367.220618:0:89766:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1784949367.220625:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949367.220627:0:89766:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949367.220629:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:0.0:1784949367.220634:0:89766:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949367.220635:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:0.0:1784949367.220639:0:89766:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:2.0:1784949367.220641:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000100:00000001:0.0:1784949367.220645:0:89766:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.220647:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3936:3936:241:4294967295] 1.2.3.4@tcp LPNI seq info [118:118:8:4294967295] 00000100:00000200:2.0:1784949367.220648:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289692288, offset 0 00000100:00000010:0.0:1784949367.220653:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bab47beb. 00000020:00000040:0.0:1784949367.220662:0:89766:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000400:00000200:2.0:1784949367.220665:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949367.220671:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 02000000:00000001:0.0:1784949367.220674:0:89766:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949367.220678:0:89766:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.220686:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:0.0:1784949367.220687:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949367.220694:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.0:1784949367.220699:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 02000000:00000001:0.0:1784949367.220700:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949367.220706:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a929270c. 00000400:00000200:2.0:1784949367.220709:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 02000000:00000010:0.0:1784949367.220713:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008b893a44. 00000800:00000200:3.0:1784949367.220720:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:0.0:1784949367.220722:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949367.220725:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000400:00000200:2.0:1784949367.220728:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643f880 00000100:00100000:0.0:1784949367.220736:0:89766:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1784949367.220744:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:2.0:1784949367.220746:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8dcd [2] + 0 00000800:00000200:3.1:1784949367.220754:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 58 secs later) 00000400:00000010:2.0:1784949367.220761:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d40cdb47. 00000400:00000200:2.0:1784949367.220769:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bf496567 00000800:00000100:3.0:1784949367.220770:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000800:00000010:3.0:1784949367.220777:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a929270c. 00000100:00000001:0.0:1784949367.220779:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1784949367.220782:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1784949367.220783:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.220783:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949367.220799:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:0.0:1784949367.220800:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bab47beb] to pc [ptlrpcd_rcv+-1] req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949367.220801:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949367.220814:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949367.220816:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bf496567 00000400:00000010:2.0:1784949367.220820:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bf496567. 00000400:00000200:3.0:1784949367.220822:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000001:0.0:1784949367.220828:0:89766:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.220828:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:3.0:1784949367.220829:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000200:2.0:1784949367.220834:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949367.220837:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1784949367.220839:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000009e6f08df x1871651289692288 msgsize 224 00000100:00000001:0.0:1784949367.220844:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:3.0:1784949367.220848:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000010:0.0:1784949367.220850:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b83ea808. 00000100:00100000:2.0:1784949367.220856:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000200:3.0:1784949367.220856:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000020:00000040:0.0:1784949367.220858:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000040:2.0:1784949367.220864:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:0.0:1784949367.220867:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1784949367.220870:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 02000000:00000001:0.0:1784949367.220872:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949367.220877:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784949367.220879:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1784949367.220880:0:337122:0:(events.c:373:request_in_callback()) Process leaving 02000000:00000010:0.0:1784949367.220883:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000820f02b3. 00000400:00000001:3.0:1784949367.220886:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949367.220889:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.220890:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:3.0:1784949367.220896:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000040:0.0:1784949367.220901:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1784949367.220903:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949367.220954:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000400:00000200:2.0:1784949367.220959:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:3.0:1784949367.220959:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000052e1e01a. 00000400:00000010:2.0:1784949367.220966:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000400:00000200:3.0:1784949367.220973:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:2.0:1784949367.220977:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949367.220982:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1784949367.220984:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00000001:0.0:1784949367.220988:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949367.220989:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1784949367.220989:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949367.220995:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:1.0:1784949367.220996:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949367.221001:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1784949367.221003:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b83ea808] to pc [ptlrpcd_00_00+0] req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949367.221012:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949367.221032:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949367.221040:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949367.221058:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1784949367.221064:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.221066:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949367.221069:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949367.221070:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1784949367.221071:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949367.221075:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949367.221079:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949367.221080:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11644) 00000100:00000001:2.0:1784949367.221080:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.221091:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306306603 00000100:00100000:1.0:1784949367.221092:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289692416:1.2.3.4@tcp:8: 00000100:00000001:1.0:1784949367.221099:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1784949367.221102:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949367.221102:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f4af3cce. 00000100:00000001:1.0:1784949367.221110:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949367.221113:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:1.0:1784949367.221114:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.221115:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00100000:0.0:1784949367.221119:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289692288 00000100:00000001:3.0:1784949367.221120:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949367.221121:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949367.221123:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949367.221126:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306306603 00000100:00000001:3.0:1784949367.221128:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949367.221129:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949367.221131:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949367.221135:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949367.221135:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f68fd7be. 00000100:00000001:1.0:1784949367.221136:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1784949367.221146:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.221147:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949367.221148:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949367.221154:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1784949367.221155:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289692288 00000100:00000001:2.0:1784949367.221157:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306306603 00000100:00000001:1.0:1784949367.221158:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949367.221159:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949367.221165:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949367.221166:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1784949367.221166:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 00000020:00000001:0.0:1784949367.221168:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949367.221174:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949367.221177:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000020:00000001:0.0:1784949367.221178:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000020:00000001:0.0:1784949367.221183:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949367.221186:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306306603 00000100:00000001:0.0:1784949367.221189:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:2.0:1784949367.221192:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000008cd34e64. 00000020:00000010:0.0:1784949367.221196:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000100:00000001:2.0:1784949367.221198:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949367.221202:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949367.221203:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949367.221205:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949367.221209:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949367.221214:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000100:00000001:2.0:1784949367.221214:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.221220:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.221222:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1784949367.221224:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949367.221228:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001c9428b0. 00000100:00000040:3.0:1784949367.221232:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949367.221233:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949367.221238:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:0.0:1784949367.221241:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000001:2.0:1784949367.221245:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949367.221251:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000010:0.0:1784949367.221255:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000075e36e9f. 00000100:00000001:2.0:1784949367.221255:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.221257:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949367.221262:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.221263:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949367.221267:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949367.221271:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949367.221273:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949367.221275:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00100000:3.0:1784949367.221277:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289692544:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949367.221286:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949367.221286:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1784949367.221291:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 02000000:00000001:3.0:1784949367.221292:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949367.221295:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949367.221296:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949367.221300:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1784949367.221304:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000052e1e01a. 00000100:00000001:0.0:1784949367.221306:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949367.221307:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949367.221318:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 00000400:00000010:3.0:1784949367.221325:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000001:0.0:1784949367.221329:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949367.221330:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289692544, portal 10 00000100:00000001:3.0:1784949367.221338:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949367.221340:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469561856 : -115544239989760 : ffff96e9c29e9c00) 00000100:00000040:3.0:1784949367.221350:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949367.221353:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949367.221358:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949367.221364:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1784949367.221369:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 837 00000100:00000040:3.0:1784949367.221375:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949367.221377:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949367.221378:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000010:3.0:1784949367.221379:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d926498d. 00000100:00000001:1.0:1784949367.221382:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949367.221386:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782153808 : -115538927397808 : ffff96eaff465450) 00000100:00000001:1.0:1784949367.221388:0:342757:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949367.221388:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289692544, offset 0 00000100:00000040:0.0:1784949367.221397:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009e6f08df x1871651289692288/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:273/0 lens 224/0 e 0 to 0 dl 1784949378 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949367.221398:0:342757:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.221406:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949367.221417:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949367.221420:0:337119:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000020:00000010:0.0:1784949367.221424:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000640e6d3c. 00000400:00000200:3.0:1784949367.221425:0:337119:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1784949367.221431:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1784949367.221435:0:337119:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:0.0:1784949367.221437:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000640e6d3c. 00000400:00000200:3.0:1784949367.221442:0:337119:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00100000:0.0:1784949367.221447:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009e6f08df pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289692288:12345-0@lo:400: 00000400:00000200:3.0:1784949367.221448:0:337119:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000200:0.0:1784949367.221457:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289692288 00000400:00000200:3.0:1784949367.221458:0:337119:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3937:3937:241:4294967295] 192.168.202.137@tcp LPNI seq info [2517:2517:8:4294967295] 00000020:00000001:0.0:1784949367.221465:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949367.221472:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949367.221477:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949367.221477:0:337119:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:0.0:1784949367.221483:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949367.221487:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:3.0:1784949367.221488:0:337119:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949367.221493:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000010:3.0:1784949367.221496:0:337119:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000a929270c. 00000020:00000001:0.0:1784949367.221497:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949367.221505:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949367.221506:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000200:3.0:1784949367.221509:0:337119:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:0.0:1784949367.221511:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949367.221517:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949367.221520:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000200:3.0:1784949367.221523:0:337119:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 02000000:00000001:0.0:1784949367.221524:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000200:3.0:1784949367.221530:0:337119:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000a929270c type 1, nob 320 niov 1 nkiov 1 02000000:00000010:0.0:1784949367.221534:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000d02c7fef. 02000000:00000001:0.0:1784949367.221539:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949367.221542:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949367.221543:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949367.221546:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289692288 00000100:00000001:3.0:1784949367.221549:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949367.221550:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949367.221557:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009e6f08df x1871651289692288/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:273/0 lens 224/224 e 0 to 0 dl 1784949378 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949367.221558:0:337119:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2260 usecs (3 jiffies) 00000100:00000001:3.0:1784949367.221565:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306306564 00000100:00000001:3.0:1784949367.221573:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306306603 00010000:00000001:0.0:1784949367.221574:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949367.221580:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306306564 00010000:00000001:0.0:1784949367.221581:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.221587:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306306603 00000100:00001000:0.0:1784949367.221589:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=89 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949367.221591:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306306564 00000100:00000001:0.0:1784949367.221596:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949367.221599:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306306603 00000100:00000040:0.0:1784949367.221601:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:0.0:1784949367.221607:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000001:3.0:1784949367.221607:0:337119:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306306564 02000000:00000001:0.0:1784949367.221613:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949367.221616:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.221616:0:337119:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306306603 00000100:00000001:0.0:1784949367.221621:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949367.221625:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1784949367.221627:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:0.0:1784949367.221633:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000100:00000001:3.0:1784949367.221634:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949367.221639:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289692288, offset 224 00000100:00000001:3.0:1784949367.221641:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949367.221648:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:3.0:1784949367.221649:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949367.221658:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949367.221659:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949367.221664:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:0.0:1784949367.221670:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643f880 00000100:00000001:3.0:1784949367.221673:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949367.221678:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949367.221682:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x97cd [1] + 224 00000100:00000001:3.0:1784949367.221683:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1784949367.221688:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949367.221692:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1784949367.221694:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000a929270c. 00000400:00000200:0.0:1784949367.221695:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949367.221704:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949367.221706:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:0.0:1784949367.221714:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1784949367.221716:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000200:0.0:1784949367.221721:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949367.221722:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d926498d 00000800:00000001:1.0:1784949367.221728:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784949367.221729:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d926498d. 00000100:00000001:2.0:1784949367.221738:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1784949367.221742:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949367.221747:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949367.221748:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949367.221759:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949367.221766:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949367.221784:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784949367.221786:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:0.0:1784949367.221790:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1784949367.221792:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949367.221801:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 00000100:00000001:2.0:1784949367.221803:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949367.221806:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000100:00000001:2.0:1784949367.221808:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1784949367.221814:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643f980 00000100:00000001:0.0:1784949367.221815:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949367.221820:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949367.221829:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949367.221830:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949367.221834:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 183152 00000100:00000001:3.0:1784949367.221835:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949367.221840:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00010000:00000001:0.0:1784949367.221848:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949367.221850:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784949367.221851:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1784949367.221853:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000800:00000001:1.0:1784949367.221853:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:3.0:1784949367.221855:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949367.221860:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000054d5ba64. 00000100:00000040:0.0:1784949367.221861:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009e6f08df x1871651289692288/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:273/0 lens 224/224 e 0 to 0 dl 1784949378 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949367.221867:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949367.221869:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000507ec8db 00000400:00000010:3.0:1784949367.221875:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000507ec8db. 00000400:00000200:1.0:1784949367.221877:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949367.221882:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949367.221893:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949367.221894:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009e6f08df pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289692288:12345-0@lo:400: Request processed in 472us (1049us total) trans 0 rc 0/0 00000400:00000200:1.0:1784949367.221897:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00100000:0.0:1784949367.221956:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 837 00000100:00000040:0.0:1784949367.221966:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:1.0:1784949367.221971:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:0.0:1784949367.221972:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000200:3.0:1784949367.221974:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949367.221979:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000200:1.0:1784949367.221979:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000010:0.0:1784949367.221986:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00000010:1.0:1784949367.221993:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005126bdfa. 00000100:00000001:3.0:1784949367.221995:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784949367.221996:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000075e36e9f. 00000100:00000001:3.0:1784949367.222000:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949367.222002:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000005126bdfa x1871651289692544 msgsize 224 00000020:00000010:0.0:1784949367.222004:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001c9428b0. 00000100:00000001:3.0:1784949367.222005:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949367.222011:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949367.222013:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:0.0:1784949367.222015:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949367.222017:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949367.222024:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00100000:1.0:1784949367.222025:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:3.0:1784949367.222025:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949367.222029:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000d02c7fef. 00000100:00001000:3.0:1784949367.222035:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=89 v=5 (5 5 5 5) 02000000:00000001:0.0:1784949367.222036:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:0.0:1784949367.222040:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00001000:3.0:1784949367.222043:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=89 v=5 (1 1 1 1) 00000100:00000200:0.0:1784949367.222046:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:3.0:1784949367.222048:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949367.222052:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949367.222053:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949367.222054:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bc6c6508. 00010000:00000001:3.0:1784949367.222062:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000400:00000010:0.0:1784949367.222065:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000120cb081. 00010000:00000001:3.0:1784949367.222066:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949367.222070:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949367.222072:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949367.222079:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949367.222080:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:3.0:1784949367.222080:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949367.222086:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289692544 02000000:00000001:2.0:1784949367.222092:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949367.222096:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949367.222098:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949367.222100:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.222102:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1784949367.222107:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.222107:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949367.222115:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289692544 00000100:00000040:3.0:1784949367.222117:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949367.222121:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949367.222125:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949367.222130:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949367.222134:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00100000:3.0:1784949367.222135:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289692288:0@lo:400: 00000020:00000001:2.0:1784949367.222139:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949367.222145:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949367.222152:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949367.222153:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949367.222159:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1784949367.222160:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b4894542. 00000100:00000040:3.0:1784949367.222164:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004614f9cb x1871651289692288/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949367.222166:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949367.222173:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b4894542. 00000020:00000010:2.0:1784949367.222183:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004565e281. 00000100:00000001:3.0:1784949367.222183:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949367.222185:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949367.222191:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000005cc3cd5d. 00000020:00000010:2.0:1784949367.222193:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000001763882a. 02000000:00000001:3.0:1784949367.222196:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:2.0:1784949367.222199:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000521c0956. 00000020:00000001:3.0:1784949367.222200:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949367.222204:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:3.0:1784949367.222208:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949367.222210:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002a753d9d. 00000100:00000040:2.0:1784949367.222212:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 02000000:00000001:3.0:1784949367.222213:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949367.222217:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1784949367.222219:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000010:3.0:1784949367.222220:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004614f9cb. 00000100:00000001:2.0:1784949367.222223:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949367.222226:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949367.222227:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949367.222230:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949367.222234:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949367.222235:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:3.0:1784949367.222239:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949367.222240:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:3.0:1784949367.222244:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.222245:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.222252:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949367.222255:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.222264:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949367.222275:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949367.222277:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:2.0:1784949367.222281:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 813 00000100:00000040:2.0:1784949367.222284:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949367.222286:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529637815424 : -115544071736192 : ffff96e9cca5f480) 00000100:00000040:2.0:1784949367.222291:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000005126bdfa x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:273/0 lens 224/0 e 0 to 0 dl 1784949378 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949367.222299:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949367.222302:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000b4894542. 00000100:00000001:2.0:1784949367.222304:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949367.222307:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000b4894542. 00000100:00100000:2.0:1784949367.222311:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000005126bdfa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289692544:12345-192.168.202.137@tcp:400: 00000100:00000200:2.0:1784949367.222315:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289692544 00000020:00000001:2.0:1784949367.222317:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949367.222320:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949367.222321:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949367.222323:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949367.222326:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:2.0:1784949367.222329:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949367.222332:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949367.222333:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949367.222335:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949367.222337:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949367.222338:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:2.0:1784949367.222341:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949367.222343:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949367.222346:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000029630d5e. 02000000:00000001:2.0:1784949367.222348:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.222350:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949367.222352:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949367.222354:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289692544 00010000:00000001:2.0:1784949367.222356:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949367.222359:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000005126bdfa x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:273/0 lens 224/224 e 0 to 0 dl 1784949378 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949367.222366:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949367.222367:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949367.222369:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=84 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949367.222372:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949367.222374:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949367.222376:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949367.222378:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949367.222380:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949367.222381:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949367.222383:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949367.222385:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000002ca344fe. 00000100:00000200:2.0:1784949367.222388:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289692544, offset 224 00000400:00000200:2.0:1784949367.222392:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949367.222399:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949367.222404:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3938:3938:241:4294967295] 192.168.202.137@tcp LPNI seq info [2518:2518:8:4294967295] 00000400:00000200:2.0:1784949367.222412:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:2.0:1784949367.222418:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949367.222422:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003cb28a6d. 00000800:00000200:2.0:1784949367.222425:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949367.222430:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949367.222433:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003cb28a6d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1784949367.222446:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949367.222448:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949367.222451:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:2.0:1784949367.222452:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949367.222454:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949367.222457:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005126bdfa x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:273/0 lens 224/224 e 0 to 0 dl 1784949378 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949367.222465:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000005126bdfa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337119:x1871651289692544:12345-192.168.202.137@tcp:400: Request processed in 165us (462us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949367.222472:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 813 00000100:00000040:2.0:1784949367.222475:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:2.0:1784949367.222478:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949367.222480:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:2.0:1784949367.222482:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000001763882a. 00000020:00000010:2.0:1784949367.222485:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000521c0956. 00000020:00000010:2.0:1784949367.222488:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004565e281. 00000020:00000040:2.0:1784949367.222493:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949367.222495:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1784949367.222554:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:2.0:1784949367.222555:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949367.222561:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003cb28a6d. 00000800:00000001:2.0:1784949367.222566:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949367.222567:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949367.222573:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949367.222579:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784949367.222580:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949367.222584:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000002ca344fe 00000400:00000010:3.0:1784949367.222590:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000002ca344fe. 00000400:00000200:2.0:1784949367.222592:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949367.222598:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949367.222603:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:2.0:1784949367.222604:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643f980 00000400:00000200:2.0:1784949367.222613:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x97d5 [1] + 224 00000800:00000001:2.0:1784949367.222620:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949367.222630:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949367.222632:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949367.222636:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949367.222640:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949367.222643:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949367.222650:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949367.222664:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949367.222668:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949367.222671:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949367.222689:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949367.222693:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949367.222697:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949367.222698:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949367.222700:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000400:00000200:3.0:1784949367.222705:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:3.0:1784949367.222707:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000100:00000001:3.0:1784949367.222709:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949367.222714:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949367.222722:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949367.222728:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949367.222730:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949367.222731:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949367.222733:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949367.222734:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949367.222735:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949367.222737:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949367.222741:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=84 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949367.222744:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=84 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949367.222746:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949367.222747:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949367.222749:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949367.222752:0:337119:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949367.222758:0:337119:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.222760:0:337119:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949367.222762:0:337119:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949367.222763:0:337119:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949367.222764:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949367.222766:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949367.222772:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949367.222774:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949367.222775:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949367.222778:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949367.222784:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:337119:1871651289692544:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949367.222788:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949367.222790:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949367.222791:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b83ea808 x1871651289692544/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949383 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949367.222797:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949367.222798:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949367.222800:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000052e1e01a. 02000000:00000001:3.0:1784949367.222802:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949367.222803:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949367.222804:0:337119:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949367.222806:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949367.222807:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000820f02b3. 02000000:00000001:3.0:1784949367.222809:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949367.222811:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949367.222812:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b83ea808. 00000100:00000001:3.0:1784949367.222814:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949367.222815:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949367.222817:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949367.222820:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949367.222821:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949367.222823:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949367.222824:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949367.263991:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306306603, at 4306306604 00000100:00000040:2.0:1784949367.264000:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306306603, at 4306306604 00000100:00000010:1.0:1784949367.264001:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f68fd7be. 00000100:00000040:3.0:1784949367.264002:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306306603, at 4306306604 00000100:00000040:0.0:1784949367.264005:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306306603, at 4306306604 00000100:00000010:2.0:1784949367.264012:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00000100:00000010:3.0:1784949367.264014:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000008cd34e64. 00000100:00000010:0.0:1784949367.264016:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f4af3cce. 00010000:00000001:2.0:1784949368.052031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949368.052040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949368.052048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949368.052050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949368.052127:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949368.052134:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949368.052138:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1784949368.243990:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1784949368.243999:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1784949368.244013:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949367/real 0] req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949368 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949368.244030:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.244033:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949368.244035:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949368.244039:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949368 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949368.244048:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.244058:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949368.244061:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.244065:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949368.244067:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949368.244070:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949368.244075:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949368 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949368.244082:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1784949368.244086:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.244088:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949368.244090:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.244093:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1784949368.244095:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1784949368.244099:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1784949368.244113:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:1.0:1784949368.244117:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1784949368.244120:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949368 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949368.244130:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289692416:1.2.3.4@tcp:8: 00000100:00000001:1.0:1784949368.244136:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.244139:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949368.244141:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000bab47beb x1871651289692416/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949368 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949368.244165:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1784949368.244170:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949368.244173:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:1.0:1784949368.244176:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949368.244182:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000008b893a44. 02000000:00000001:1.0:1784949368.244192:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1784949368.244194:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1784949368.244195:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949368.244201:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000bab47beb. 00000100:00000001:2.0:1784949368.244211:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:1.0:1784949368.244213:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949368.244217:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.244220:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1784949368.244222:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f37d62c3. 00000100:00000001:1.0:1784949368.244224:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949368.244231:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949368.244233:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:1.0:1784949368.244237:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949368.244241:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.244243:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949368.244248:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949368.244255:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949368.244265:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003cb28a6d. 00000100:00000001:2.0:1784949368.244274:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949368.244283:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949368.244297:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949368.244300:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949368.244308:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f37d62c3] to pc [ptlrpcd_00_01+1] req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.244356:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949368.244357:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949368.244364:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:3.0:1784949368.244366:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949368.244374:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1784949368.244375:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1784949368.244382:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1784949368.244385:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1784949368.244388:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949368.244392:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1784949368.244401:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11614 00000100:00000001:0.0:1784949368.244404:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949368.244412:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.244417:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1784949368.244419:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:3.0:1784949368.244423:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949368.244429:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11639 00000400:00000200:2.0:1784949368.244448:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000400:00000200:2.0:1784949368.244452:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00080000:2.0:1784949368.244456:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1784949368.244459:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949368.244463:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949368.244467:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949368.244469:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:2.0:1784949368.244490:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949368.244493:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:2.0:1784949368.244495:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949368.244499:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000020:00000001:2.0:1784949368.244501:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:2.0:1784949368.244504:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000040:2.0:1784949368.244508:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 00000100:00000001:2.0:1784949368.244510:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1784949368.244512:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000040:2.0:1784949368.244514:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00000100:00000001:2.0:1784949368.244520:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000200:1.0:1784949368.244524:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000020:00000040:2.0:1784949368.244526:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 00000100:00080000:2.0:1784949368.244534:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000400:00000010:1.0:1784949368.244535:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000100:00080000:2.0:1784949368.244539:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:2.0:1784949368.244545:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949368.244546:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 02000000:00000001:2.0:1784949368.244550:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1784949368.244556:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949368.244558:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1784949368.244559:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000056670828. 02000000:00000001:3.0:1784949368.244562:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949368.244569:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949368.244570:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.244576:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000001:2.0:1784949368.244578:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:3.0:1784949368.244582:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000f37d62c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289692672:0@lo:400: 00000100:00000001:2.0:1784949368.244584:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.244591:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1784949368.244594:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003dd0d971. 00000400:00000200:1.0:1784949368.244596:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000020:00000040:2.0:1784949368.244601:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:3.0:1784949368.244601:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1784949368.244605:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1784949368.244607:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949368.244610:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1784949368.244612:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1784949368.244615:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.244616:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000010:3.0:1784949368.244620:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f7a7941e. 00000100:00000001:2.0:1784949368.244624:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949368.244625:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.244627:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1784949368.244630:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1784949368.244637:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000005468aedd. 02000000:00000001:2.0:1784949368.244638:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.244639:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 00000400:00000200:1.0:1784949368.244652:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3939:3939:241:4294967295] 1.2.3.4@tcp LPNI seq info [119:119:8:4294967295] 02000000:00000010:2.0:1784949368.244652:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000b1427c95. 00000400:00000010:3.0:1784949368.244653:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b61e4a12. 00000100:00000001:2.0:1784949368.244661:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949368.244665:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289692672, portal 25 00000100:00100000:2.0:1784949368.244671:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949368.244671:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1784949368.244674:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00000001:3.0:1784949368.244675:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469571968 : -115544239979648 : ffff96e9c29ec380) 00000100:00000001:2.0:1784949368.244688:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949368.244693:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.244699:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000040:2.0:1784949368.244702:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dd0d971] to pc [ptlrpcd_rcv+-1] req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949368.244719:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000800:00000010:1.0:1784949368.244725:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076916763. 00000100:00000001:2.0:1784949368.244730:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1784949368.244736:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000040:2.0:1784949368.244737:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1784949368.244746:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 00000100:00000001:2.0:1784949368.244747:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949368.244753:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a6696f47. 00000020:00000040:2.0:1784949368.244758:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000800:00000200:1.1:1784949368.244759:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:0.0:1784949368.244761:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949368.244762:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1784949368.244769:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:1.1:1784949368.244769:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 57 secs later) 02000000:00000001:2.0:1784949368.244770:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1784949368.244775:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1784949368.244776:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1784949368.244783:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000100:00000040:0.0:1784949368.244785:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1784949368.244787:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000284ae7c9. 00000800:00000010:1.0:1784949368.244788:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000076916763. 00000100:00000001:2.0:1784949368.244797:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.244799:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1784949368.244810:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1784949368.244817:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000040:3.0:1784949368.244823:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949368.244831:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949368.244843:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000001:0.0:1784949368.244854:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1784949368.244856:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000001:0.0:1784949368.244860:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949368.244869:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784949368.244873:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 02000000:00000001:0.0:1784949368.244875:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949368.244882:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1784949368.244884:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.244884:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000001:2.0:1784949368.244886:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949368.244947:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003dd0d971 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289692800:192.168.202.137@tcp:8: 00000100:00000040:2.0:1784949368.244947:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a6696f47] to pc [ptlrpcd_00_02+2] req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949368.244948:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000001:1.0:1784949368.244962:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784949368.244964:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000001:1.0:1784949368.244967:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1784949368.244969:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1784949368.244970:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1784949368.244974:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 02000000:00000001:0.0:1784949368.244976:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949368.244981:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000040:3.0:1784949368.244983:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:0.0:1784949368.244985:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949368.244991:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000010:3.0:1784949368.244993:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 02000000:00000010:0.0:1784949368.244995:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d02c7fef. 00000400:00000200:1.0:1784949368.245001:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00000200:3.0:1784949368.245001:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289692672, offset 0 02000000:00000001:0.0:1784949368.245002:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.245008:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949368.245010:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000005ae2686. 00000400:00000010:0.0:1784949368.245013:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000047084e78. 00000100:00000040:2.0:1784949368.245017:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11645) 00000400:00000200:3.0:1784949368.245021:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:1.0:1784949368.245022:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000400:00000200:1.0:1784949368.245030:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000400:00000200:1.0:1784949368.245035:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000400:00000010:0.0:1784949368.245036:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000000226c39. 00000400:00000200:3.0:1784949368.245044:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000200:0.0:1784949368.245045:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289692800, portal 4 00000100:00000001:0.0:1784949368.245053:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949368.245055:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469585408 : -115544239966208 : ffff96e9c29ef800) 00000100:00000040:0.0:1784949368.245066:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949393 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949368.245068:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643fa00 00000100:00000001:1.0:1784949368.245073:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949368.245081:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949368.245083:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949368.245085:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1784949368.245088:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8e59 [2] + 0 00000100:00000040:0.0:1784949368.245096:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000040:1.0:1784949368.245097:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949368.245099:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001004a12d. 00000400:00000010:0.0:1784949368.245105:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000400:00000200:3.0:1784949368.245109:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000092ad473a 00000100:00000200:0.0:1784949368.245117:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289692800, offset 0 00000100:00000001:1.0:1784949368.245119:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1784949368.245121:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949368.245123:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949368.245130:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1784949368.245131:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949368.245132:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:1.0:1784949368.245136:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949368.245142:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.245144:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1784949368.245151:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a6696f47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289692928:192.168.202.137@tcp:400: 00000100:00000001:2.0:1784949368.245168:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949368.245169:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:1.0:1784949368.245184:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:0.0:1784949368.245185:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1784949368.245193:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:0.0:1784949368.245195:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:1.0:1784949368.245197:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949368.245205:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1784949368.245206:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784949368.245213:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000010:1.0:1784949368.245215:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 02000000:00000001:1.0:1784949368.245223:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949368.245224:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3940:3940:241:4294967295] 192.168.202.137@tcp LPNI seq info [2519:2519:8:4294967295] 00000400:00000010:1.0:1784949368.245232:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000877cac20. 00000400:00000010:1.0:1784949368.245242:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000400:00000200:0.0:1784949368.245250:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000200:1.0:1784949368.245252:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289692928, portal 10 00000100:00000001:1.0:1784949368.245261:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949368.245265:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469583616 : -115544239968000 : ffff96e9c29ef100) 00000400:00000200:3.0:1784949368.245268:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000092ad473a 00000800:00000200:0.0:1784949368.245273:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949368.245277:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000092ad473a. 00000800:00000010:0.0:1784949368.245282:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000662152ce. 00000100:00000040:1.0:1784949368.245284:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.245289:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1784949368.245295:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1784949368.245299:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:3.0:1784949368.245308:0:337120:0:(events.c:337:request_in_callback()) incoming req@00000000f7830c1e x1871651289692672 msgsize 224 00000800:00000200:0.0:1784949368.245322:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00100000:3.0:1784949368.245330:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000800:00000200:0.0:1784949368.245335:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000662152ce type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1784949368.245335:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949368.245337:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000040:1.0:1784949368.245348:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949368.245354:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000100:00000200:1.0:1784949368.245357:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289692928, offset 0 00000100:00000001:3.0:1784949368.245362:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:0.0:1784949368.245366:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.245369:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000040:0.0:1784949368.245375:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000400:00000200:3.0:1784949368.245376:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:3.0:1784949368.245383:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000400:00000200:1.0:1784949368.245385:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949368.245386:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306307627 00000100:00000001:3.0:1784949368.245392:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:0.0:1784949368.245402:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d0a848b. 00000400:00000200:1.0:1784949368.245403:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784949368.245408:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000200:3.0:1784949368.245408:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949368.245415:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289692672 00000400:00000200:1.0:1784949368.245417:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949368.245418:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 0 02000000:00000001:2.0:1784949368.245424:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949368.245447:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:1.0:1784949368.245448:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949368.245450:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306307627 00000100:00000001:2.0:1784949368.245455:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.245460:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000010:0.0:1784949368.245461:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e631904c. 02000000:00000001:2.0:1784949368.245464:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.245464:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949368.245473:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 1 00000100:00000040:3.0:1784949368.245473:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949368.245474:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3941:3941:240:4294967295] 192.168.202.137@tcp LPNI seq info [2520:2520:7:4294967295] 00000100:00100000:2.0:1784949368.245475:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289692672 00000100:00000001:0.0:1784949368.245483:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306307627 00000020:00000001:2.0:1784949368.245484:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949368.245489:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000010:0.0:1784949368.245495:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f9ed4c5a. 00000020:00000001:2.0:1784949368.245496:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949368.245500:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949368.245503:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:0.0:1784949368.245506:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949368.245507:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1784949368.245514:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949368.245516:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306307627 00000400:00000200:1.0:1784949368.245516:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:2.0:1784949368.245521:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000010:0.0:1784949368.245527:0:337118:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005edff372. 00000100:00000001:2.0:1784949368.245530:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000800:00000200:1.0:1784949368.245537:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949368.245538:0:337118:0:(niobuf.c:678:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request now active for cpu 3 00000020:00000010:2.0:1784949368.245539:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:3.0:1784949368.245539:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949368.245548:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1784949368.245549:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000076916763. 00000100:00000001:2.0:1784949368.245551:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949368.245552:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306307627 00000100:00000001:0.0:1784949368.245554:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949368.245564:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000800:00000200:1.0:1784949368.245565:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949368.245566:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.245566:0:337120:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, keep current deadline 4306307627 00000100:00000001:3.0:1784949368.245576:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306307627 00000100:00000001:0.0:1784949368.245577:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784949368.245581:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004565e281. 00000800:00000200:1.0:1784949368.245587:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784949368.245588:0:337120:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, keep current deadline 4306307627 00000100:00000001:0.0:1784949368.245591:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784949368.245592:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000001763882a. 00000100:00000001:3.0:1784949368.245595:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306307627 00000100:00000001:0.0:1784949368.245598:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:1.0:1784949368.245599:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000076916763 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949368.245602:0:337120:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, keep current deadline 4306307627 00000020:00000010:2.0:1784949368.245604:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000521c0956. 00000100:00000001:0.0:1784949368.245611:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949368.245613:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306307627 00000100:00000001:0.0:1784949368.245616:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949368.245619:0:337120:0:(niobuf.c:692:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, keep current deadline 4306307627 00000100:00000040:2.0:1784949368.245621:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949368.245622:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.245623:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949368.245625:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.245629:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949368.245634:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.245634:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.245635:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:1.0:1784949368.245637:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2260 usecs (3 jiffies) 00000100:00000001:2.0:1784949368.245642:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.245645:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.245666:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306307588 00000100:00000001:3.0:1784949368.245668:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949368.245673:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.245678:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306307627 00000100:00000001:3.0:1784949368.245680:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949368.245684:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949368.245687:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306307588 00000100:00000001:2.0:1784949368.245689:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.245690:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949368.245695:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306307627 00000100:00000001:3.0:1784949368.245696:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949368.245702:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.245704:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306307588 00000100:00000001:3.0:1784949368.245707:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.245709:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949368.245713:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306307627 00000100:00000001:2.0:1784949368.245714:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:1.0:1784949368.245722:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306307588 00000100:00100000:2.0:1784949368.245723:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 838 00000100:00000001:1.0:1784949368.245731:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306307627 00000100:00000040:2.0:1784949368.245731:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949368.245739:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949368.245739:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782159952 : -115538927391664 : ffff96eaff466c50) 00000100:00000001:1.0:1784949368.245748:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949368.245753:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f7830c1e x1871651289692672/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:274/0 lens 224/0 e 0 to 0 dl 1784949379 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1784949368.245754:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1784949368.245755:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.245764:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1784949368.245765:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000662152ce. 00000100:00000001:2.0:1784949368.245776:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949368.245777:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949368.245780:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:2.0:1784949368.245781:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000070051d11. 00000100:00000001:1.0:1784949368.245784:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949368.245790:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949368.245791:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949368.245793:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1784949368.245799:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000070051d11. 00000100:00000001:1.0:1784949368.245800:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949368.245802:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000400:00000010:0.0:1784949368.245807:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:1.0:1784949368.245808:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949368.245813:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949368.245815:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:2.0:1784949368.245816:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f7830c1e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289692672:12345-0@lo:400: 00000100:00000001:1.0:1784949368.245817:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949368.245820:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000076916763. 00000100:00000200:0.0:1784949368.245825:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949393 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949368.245831:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289692672 00000400:00000200:3.0:1784949368.245832:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949368.245838:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949368.245845:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:3.0:1784949368.245850:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949368.245851:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949368.245852:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949368.245853:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949368.245861:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000040:0.0:1784949368.245864:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949393 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949368.245864:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000800:00000001:1.0:1784949368.245865:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949368.245866:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000400:00000010:3.0:1784949368.245871:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000800:00000001:1.0:1784949368.245875:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949368.245877:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:3.0:1784949368.245881:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000800:00000001:1.0:1784949368.245883:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949368.245884:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949368.245893:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949368.245894:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:3.0:1784949368.245953:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1784949368.245955:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949368.245955:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000020:00000001:2.0:1784949368.245960:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000040:3.0:1784949368.245963:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949368.245964:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1784949368.245966:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949368.245972:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:0.0:1784949368.245979:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949368.245980:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:1.0:1784949368.245981:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c643fa80 02000000:00000001:2.0:1784949368.245987:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000800:00000001:0.0:1784949368.245992:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.245994:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949368.245995:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000bcfc6676. 00000800:00000001:0.0:1784949368.245996:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000100:1.0:1784949368.246002:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289692800 offset 0 length 520: 4 00000100:00000001:3.0:1784949368.246003:0:337114:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1784949368.246004:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949368.246005:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949368.246016:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000f01f39bd is in inconsistent state, don't perform health checking (-2, 0) 00000100:00000001:2.0:1784949368.246016:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949368.246025:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949368.246027:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949368.246027:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00010000:00000040:2.0:1784949368.246030:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289692672 00010000:00000001:2.0:1784949368.246038:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949368.246045:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f7830c1e x1871651289692672/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:274/0 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949368.246052:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643fb00 00000800:00000001:1.0:1784949368.246054:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000001:2.0:1784949368.246067:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:0.0:1784949368.246069:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 183376 00010000:00000001:2.0:1784949368.246071:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949368.246072:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949368.246082:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949368.246083:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00001000:2.0:1784949368.246083:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=90 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949368.246092:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:1.0:1784949368.246094:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949368.246095:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949368.246100:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:2.0:1784949368.246104:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000800:00000001:0.0:1784949368.246105:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1784949368.246107:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949368.246111:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949368.246112:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949368.246122:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:0.0:1784949368.246123:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949368.246132:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000400:00000010:2.0:1784949368.246133:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000100:00000200:0.0:1784949368.246134:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:2.0:1784949368.246139:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289692672, offset 224 00000100:00000010:0.0:1784949368.246158:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000021c72e86. 00000100:00000040:0.0:1784949368.246164:0:337112:0:(events.c:337:request_in_callback()) incoming req@0000000021c72e86 x1871651289692928 msgsize 224 00000400:00000200:2.0:1784949368.246171:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:0.0:1784949368.246180:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000400:00000200:2.0:1784949368.246187:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949368.246195:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643fa00 00000100:00000001:0.0:1784949368.246198:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:2.0:1784949368.246203:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x97ed [1] + 224 00000400:00000200:2.0:1784949368.246211:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949368.246215:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949368.246216:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:0.0:1784949368.246224:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949368.246226:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949368.246234:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949368.246246:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.246252:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949368.246257:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289692928 02000000:00000001:3.0:1784949368.246260:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949368.246261:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949368.246263:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949368.246266:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.246267:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1784949368.246270:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289692928 00000020:00000001:3.0:1784949368.246274:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1784949368.246277:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000020:00000040:3.0:1784949368.246278:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000010:2.0:1784949368.246283:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000020:00000001:3.0:1784949368.246284:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1784949368.246310:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000040:3.0:1784949368.246310:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784949368.246316:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:3.0:1784949368.246318:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949368.246323:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:2.0:1784949368.246325:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.246330:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784949368.246334:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949368.246335:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:3.0:1784949368.246339:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000005451e184. 00000100:00000001:1.0:1784949368.246342:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1784949368.246343:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949368.246346:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1784949368.246350:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949368.246352:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000005451e184. 00000400:00000001:1.0:1784949368.246353:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1784949368.246359:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:1.0:1784949368.246361:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949368.246367:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005468aedd. 00000020:00000010:3.0:1784949368.246367:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000052e1e01a. 00000100:00000040:2.0:1784949368.246368:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f7830c1e x1871651289692672/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:274/0 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949368.246379:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000079fab95b. 00000400:00000200:1.0:1784949368.246380:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b61e4a12 00000400:00000010:1.0:1784949368.246387:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b61e4a12. 00000020:00000010:3.0:1784949368.246392:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:1.0:1784949368.246395:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1784949368.246399:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f7830c1e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289692672:12345-0@lo:400: Request processed in 615us (1083us total) trans 0 rc 0/0 00000100:00000040:3.0:1784949368.246408:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000200:1.0:1784949368.246409:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.246417:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00100000:2.0:1784949368.246419:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 838 00000100:00000001:3.0:1784949368.246424:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:2.0:1784949368.246430:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949368.246432:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949368.246456:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949368.246462:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949368.246464:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000200:1.0:1784949368.246466:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.246469:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000010:2.0:1784949368.246470:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000001763882a. 00000100:00000001:3.0:1784949368.246474:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949368.246481:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000521c0956. 00000100:00000001:1.0:1784949368.246492:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:2.0:1784949368.246493:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004565e281. 00000100:00000001:1.0:1784949368.246496:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.246499:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.246503:0:337120:0:(client.c:1580:after_reply()) Process entered 00000020:00000040:2.0:1784949368.246503:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:1.0:1784949368.246509:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:2.0:1784949368.246512:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:1.0:1784949368.246513:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949368.246513:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000010:2.0:1784949368.246517:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 00000000bcfc6676. 00000100:00000001:1.0:1784949368.246519:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.246520:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:2.0:1784949368.246522:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:1.0:1784949368.246527:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949368.246528:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949368.246536:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 814 00000100:00001000:1.0:1784949368.246537:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=90 v=5 (5 5 5 5) 00000100:00000200:2.0:1784949368.246540:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000040:3.0:1784949368.246546:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00001000:1.0:1784949368.246549:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=90 v=5 (1 1 1 1) 00000400:00000010:2.0:1784949368.246550:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000009129b91b. 00000100:00000001:3.0:1784949368.246552:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469563648 : -115544239987968 : ffff96e9c29ea300) 00000100:00000001:1.0:1784949368.246563:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000400:00000010:2.0:1784949368.246564:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fea29ebd. 00000100:00000040:3.0:1784949368.246567:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000021c72e86 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:274/0 lens 224/0 e 0 to 0 dl 1784949379 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949368.246569:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949368.246577:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1784949368.246583:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.246592:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.246593:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949368.246601:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949368.246601:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000005451e184. 00000100:00000001:3.0:1784949368.246609:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949368.246616:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000005451e184. 00000100:00000001:1.0:1784949368.246627:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1784949368.246628:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000021c72e86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289692928:12345-192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949368.246637:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000200:3.0:1784949368.246638:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289692928 00000100:00000001:1.0:1784949368.246644:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949368.246645:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949368.246668:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949368.246672:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949368.246673:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949368.246678:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:3.0:1784949368.246682:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:3.0:1784949368.246689:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:3.0:1784949368.246694:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949368.246700:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:3.0:1784949368.246704:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949368.246707:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949368.246712:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:3.0:1784949368.246718:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949368.246723:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:1.0:1784949368.246726:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000f37d62c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289692672:0@lo:400: 02000000:00000010:3.0:1784949368.246728:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008618f1b2. 02000000:00000001:3.0:1784949368.246733:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.246740:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.246743:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949368.246743:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.246752:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000040:3.0:1784949368.246752:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289692928 00010000:00000001:3.0:1784949368.246758:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000040:1.0:1784949368.246761:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000f37d62c3 x1871651289692672/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:3.0:1784949368.246766:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000021c72e86 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:274/0 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949368.246804:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:3.0:1784949368.246806:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 02000000:00000001:1.0:1784949368.246809:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00010000:00000001:3.0:1784949368.246810:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949368.246814:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f7a7941e. 00000100:00001000:3.0:1784949368.246815:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=85 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949368.246823:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949368.246825:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949368.246826:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000001:1.0:1784949368.246830:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784949368.246832:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:1.0:1784949368.246834:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 02000000:00000001:3.0:1784949368.246838:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:1.0:1784949368.246839:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:3.0:1784949368.246842:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949368.246844:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003cb28a6d. 00000100:00000001:3.0:1784949368.246848:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949368.246850:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949368.246855:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:3.0:1784949368.246855:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000010:1.0:1784949368.246858:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000f37d62c3. 00000400:00000010:3.0:1784949368.246860:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007f20cd9a. 00000100:00000001:1.0:1784949368.246863:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949368.246866:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949368.246867:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289692928, offset 224 00000100:00000001:1.0:1784949368.246869:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949368.246878:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1784949368.246879:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949368.246882:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949368.246885:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949368.246888:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949368.246893:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949368.246958:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3942:3942:241:4294967295] 192.168.202.137@tcp LPNI seq info [2521:2521:8:4294967295] 00000400:00000200:3.0:1784949368.246969:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949368.246975:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949368.246979:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949368.246983:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949368.246988:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:3.0:1784949368.246991:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949368.247004:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949368.247007:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949368.247009:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949368.247011:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949368.247013:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949368.247017:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000021c72e86 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:274/0 lens 224/224 e 0 to 0 dl 1784949379 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949368.247027:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000021c72e86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289692928:12345-192.168.202.137@tcp:400: Request processed in 433us (861us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949368.247036:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 814 00000100:00000040:3.0:1784949368.247038:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949368.247040:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949368.247042:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949368.247045:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000079fab95b. 00000020:00000010:3.0:1784949368.247049:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949368.247052:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000052e1e01a. 00000020:00000040:3.0:1784949368.247057:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949368.247060:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949368.247082:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949368.247087:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:0.0:1784949368.247093:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949368.247099:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949368.247104:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007f20cd9a 00000400:00000010:0.0:1784949368.247106:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007f20cd9a. 00000100:00000001:0.0:1784949368.247110:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949368.247112:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949368.247114:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949368.247124:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949368.247127:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949368.247130:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949368.247138:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:1.0:1784949368.247162:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643fb00 00000400:00000200:1.0:1784949368.247169:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x97fd [1] + 224 00000800:00000001:1.0:1784949368.247174:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949368.247178:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949368.247180:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949368.247184:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949368.247189:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949368.247193:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949368.247202:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949368.247223:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1784949368.247231:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949368.247235:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.247268:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949368.247272:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1784949368.247276:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1784949368.247278:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949368.247280:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000877cac20. 00000400:00000200:3.0:1784949368.247284:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000400:00000010:3.0:1784949368.247286:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000100:00000001:3.0:1784949368.247289:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949368.247293:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949368.247302:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.247309:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1784949368.247311:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.247313:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949368.247314:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949368.247316:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1784949368.247318:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949368.247320:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949368.247325:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=85 v=5 (5 5 5 5) 00000100:00001000:3.0:1784949368.247328:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=85 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949368.247330:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949368.247332:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1784949368.247333:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1784949368.247337:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949368.247344:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.247346:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1784949368.247348:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:3.0:1784949368.247350:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1784949368.247369:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949368.247371:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.247378:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949368.247380:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949368.247381:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949368.247384:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949368.247390:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a6696f47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289692928:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949368.247393:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.247395:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949368.247397:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a6696f47 x1871651289692928/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949368.247403:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1784949368.247404:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949368.247406:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:3.0:1784949368.247408:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1784949368.247409:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:3.0:1784949368.247411:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:3.0:1784949368.247414:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949368.247415:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000284ae7c9. 02000000:00000001:3.0:1784949368.247418:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949368.247419:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949368.247421:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a6696f47. 00000100:00000001:3.0:1784949368.247424:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949368.247425:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.247426:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949368.247430:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949368.247431:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949368.247433:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949368.247434:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949368.287975:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306307627, at 4306307628 00000100:00000010:1.0:1784949368.287987:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000e631904c. 00000100:00000040:2.0:1784949368.288016:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306307627, at 4306307628 00000100:00000040:3.0:1784949368.288017:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306307627, at 4306307628 00000100:00000040:0.0:1784949368.288029:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306307627, at 4306307628 00000100:00000010:2.0:1784949368.288031:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f9ed4c5a. 00000100:00000010:3.0:1784949368.288034:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005edff372. 00000100:00000010:0.0:1784949368.288042:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d0a848b. 00010000:00000001:2.0:1784949369.076035:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949369.076045:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949369.076055:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949369.076057:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949369.076160:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949369.076164:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949369.076167:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949370.100046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949370.100055:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949370.100064:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949370.100066:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949370.100158:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949370.100165:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949370.100170:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949371.124094:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949371.124103:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949371.124112:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949371.124113:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949371.124251:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949371.124257:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949371.124262:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949372.148101:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949372.148112:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949372.148120:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949372.148125:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949372.148128:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949372.148132:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949372.148135:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949372.148138:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949372.148141:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949372.148145:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949372.148293:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949372.148299:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949372.148304:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949373.172039:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949373.172049:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949373.172059:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949373.172061:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949373.172127:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949373.172132:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949373.172135:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949373.364109:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949373.364132:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949373.364137:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949373.364147:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949373.364153:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949373.364160:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949373.364164:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949373.364166:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949373.364183:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1784949373.364198:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006b16a92e. 00000020:00000040:3.0:1784949373.364210:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:3.0:1784949373.364219:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949373.364225:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949373.364234:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949373.364248:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000005f459fde. 00000100:00000001:3.0:1784949373.364260:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949373.364276:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1784949373.364299:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1784949373.364304:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949373.364313:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b16a92e] to pc [ptlrpcd_00_03+3] req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949373.364379:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1784949373.364385:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:3.0:1784949373.364393:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1784949373.364399:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1784949373.364407:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:2.0:1784949373.364410:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1784949373.364417:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000eb4f6eba. 00000100:00000001:2.0:1784949373.364419:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949373.364427:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949373.364429:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1784949373.364430:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000040:2.0:1784949373.364439:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949373.364440:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949373.364442:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1784949373.364450:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1784949373.364459:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.364464:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949373.364469:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949373.364472:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000fc2fde8d. 02000000:00000001:2.0:1784949373.364475:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949373.364484:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949373.364485:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949373.364489:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949373.364498:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289693056:0@lo:400: 00000100:00000040:3.0:1784949373.364508:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949373.364509:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1784949373.364519:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1784949373.364523:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949373.364532:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1784949373.364544:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:2.0:1784949373.364545:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004565e281. 00000100:00000001:3.0:1784949373.364551:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949373.364552:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949373.364569:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000068d5a5a2. 00000100:00000040:3.0:1784949373.364571:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb4f6eba] to pc [ptlrpcd_00_00+0] req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949373.364581:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000684e85cd. 00000100:00000200:2.0:1784949373.364598:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693056, portal 25 00000100:00000001:2.0:1784949373.364604:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1784949373.364606:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529482521600 : -115544227030016 : ffff96e9c3645c00) 00000100:00000040:2.0:1784949373.364615:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949373.364627:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1784949373.364628:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949373.364634:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949373.364641:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000100:00000200:2.0:1784949373.364646:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289693056, offset 0 00000100:00000040:3.0:1784949373.364646:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11650) 00000100:00000001:1.0:1784949373.364662:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949373.364669:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00100000:1.0:1784949373.364672:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1784949373.364678:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949373.364686:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949373.364690:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949373.364697:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949373.364699:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1784949373.364700:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949373.364708:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949373.364710:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949373.364714:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1784949373.364725:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643fb80 00000100:00000040:1.0:1784949373.364729:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949373.364745:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8e79 [2] + 0 00000100:00000001:1.0:1784949373.364760:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000010:2.0:1784949373.364762:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000009a55be35. 00000100:00000001:1.0:1784949373.364765:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949373.364773:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000ba56b0c8 02000000:00000001:1.0:1784949373.364774:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949373.364780:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949373.364786:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949373.364789:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00100000:1.0:1784949373.364791:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289693184:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949373.364805:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1784949373.364807:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 02000000:00000001:1.0:1784949373.364813:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949373.364821:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949373.364821:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ba56b0c8 02000000:00000001:1.0:1784949373.364833:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000010:2.0:1784949373.364833:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000ba56b0c8. 02000000:00000010:1.0:1784949373.364843:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 00000100:00000001:2.0:1784949373.364847:0:337122:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:1.0:1784949373.364888:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949373.364901:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000200:2.0:1784949373.364946:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:2.0:1784949373.364956:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000002b339182 x1871651289693056 msgsize 224 00000400:00000010:1.0:1784949373.364963:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00100000:2.0:1784949373.364976:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000200:1.0:1784949373.364979:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693184, portal 10 00000100:00000040:2.0:1784949373.364983:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:1.0:1784949373.364989:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949373.364995:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529549148160 : -115544160403456 : ffff96e9c75d0000) 00000100:00000001:2.0:1784949373.365008:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000040:1.0:1784949373.365012:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949373.365025:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000100:00000001:1.0:1784949373.365027:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:2.0:1784949373.365034:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000100:00000040:1.0:1784949373.365040:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949373.365047:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000100:00000001:2.0:1784949373.365049:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1784949373.365053:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289693184, offset 0 00000100:00000200:2.0:1784949373.365062:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949373.365069:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949373.365080:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784949373.365090:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00100000:0.0:1784949373.365091:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693056 00000100:00000001:2.0:1784949373.365095:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784949373.365100:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1784949373.365102:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000040:2.0:1784949373.365108:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949373.365111:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:0.0:1784949373.365112:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949373.365123:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949373.365124:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949373.365137:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 02000000:00000001:0.0:1784949373.365141:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949373.365154:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3943:3943:241:4294967295] 192.168.202.137@tcp LPNI seq info [2522:2522:8:4294967295] 00000100:00100000:0.0:1784949373.365157:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693056 00000100:00000001:2.0:1784949373.365160:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949373.365169:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949373.365172:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:0.0:1784949373.365177:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949373.365187:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1784949373.365188:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000040:0.0:1784949373.365198:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000800:00000200:1.0:1784949373.365204:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:0.0:1784949373.365212:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000010:1.0:1784949373.365215:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ca1124f5. 00000100:00000001:2.0:1784949373.365222:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949373.365226:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:2.0:1784949373.365233:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306312747 00000800:00000200:1.0:1784949373.365235:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949373.365239:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000010:2.0:1784949373.365248:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000078bb23fa. 00000020:00000010:0.0:1784949373.365254:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000800:00000200:1.0:1784949373.365261:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784949373.365262:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1784949373.365266:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1784949373.365272:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000ca1124f5 type 1, nob 320 niov 1 nkiov 1 00000020:00000010:0.0:1784949373.365275:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000001:2.0:1784949373.365285:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306312747 00000020:00000010:0.0:1784949373.365293:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001c9428b0. 00000100:00000010:2.0:1784949373.365295:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000824e9543. 00000100:00000001:2.0:1784949373.365307:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949373.365309:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949373.365313:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000001:2.0:1784949373.365313:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306312747 00000100:00000040:1.0:1784949373.365318:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2260 usecs (3 jiffies) 00000100:00000010:2.0:1784949373.365327:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000100:00000001:1.0:1784949373.365329:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306312708 00000020:00000010:0.0:1784949373.365333:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000075e36e9f. 00000100:00000001:1.0:1784949373.365348:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306312747 00000100:00000001:2.0:1784949373.365349:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1784949373.365359:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306312708 00000100:00000001:2.0:1784949373.365361:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306312747 00000100:00000001:1.0:1784949373.365366:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306312747 00000100:00000010:2.0:1784949373.365371:0:337122:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002eb614c3. 00000100:00000001:1.0:1784949373.365373:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306312708 00000100:00000040:0.0:1784949373.365374:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949373.365382:0:337122:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1784949373.365384:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306312747 00000100:00000001:0.0:1784949373.365388:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949373.365393:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306312708 00000100:00000001:2.0:1784949373.365394:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949373.365396:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:1.0:1784949373.365398:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306312747 00000100:00000001:2.0:1784949373.365406:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949373.365407:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949373.365408:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949373.365414:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949373.365420:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.365422:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949373.365429:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949373.365432:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.365434:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949373.365437:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.365441:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949373.365448:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949373.365450:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949373.365454:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.365458:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949373.365468:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949373.365496:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949373.365501:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949373.365511:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 839 00000100:00000040:0.0:1784949373.365517:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:0.0:1784949373.365522:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782154832 : -115538927396784 : ffff96eaff465850) 00000100:00000040:0.0:1784949373.365531:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000002b339182 x1871651289693056/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:279/0 lens 224/0 e 0 to 0 dl 1784949384 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949373.365550:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949373.365557:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:0.0:1784949373.365562:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949373.365566:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00100000:0.0:1784949373.365575:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000002b339182 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289693056:12345-0@lo:400: 00000100:00000200:0.0:1784949373.365583:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693056 00000020:00000001:0.0:1784949373.365589:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949373.365596:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949373.365601:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949373.365606:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949373.365612:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949373.365619:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000001:1.0:1784949373.365622:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:0.0:1784949373.365624:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949373.365633:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949373.365638:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000200:3.0:1784949373.365640:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1784949373.365641:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949373.365644:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949373.365654:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:3.0:1784949373.365656:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ca1124f5. 00000020:00000001:0.0:1784949373.365657:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000800:00000001:1.0:1784949373.365661:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949373.365667:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000400:00000200:3.0:1784949373.365672:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:0.0:1784949373.365680:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:1.0:1784949373.365680:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949373.365695:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000010:0.0:1784949373.365697:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000032060ac0. 00000400:00000200:3.0:1784949373.365706:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000200:1.0:1784949373.365707:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643fc00 02000000:00000001:0.0:1784949373.365709:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949373.365717:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000100:00000001:0.0:1784949373.365723:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949373.365730:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1784949373.365732:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949373.365740:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 183600 00010000:00000040:0.0:1784949373.365745:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693056 00000100:00000200:3.0:1784949373.365748:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949373.365758:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000800:00000001:1.0:1784949373.365767:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000200:0.0:1784949373.365774:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000002b339182 x1871651289693056/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:279/0 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949373.365796:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949373.365810:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949373.365834:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000001:0.0:1784949373.365845:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949373.365852:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:0.0:1784949373.365853:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949373.365885:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949373.365895:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00001000:0.0:1784949373.365901:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=95 v=5 (1 1 1 1) 00000100:00000040:3.0:1784949373.365953:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949373.365975:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1784949373.365982:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949373.365984:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000000115b4b. 00000100:00000040:0.0:1784949373.365987:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:3.0:1784949373.365993:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784949373.365996:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000000115b4b x1871651289693184 msgsize 224 00000100:00000001:0.0:1784949373.365998:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 02000000:00000001:0.0:1784949373.366010:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000100:00100000:1.0:1784949373.366012:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:0.0:1784949373.366018:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949373.366026:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949373.366033:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000001:1.0:1784949373.366036:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:0.0:1784949373.366042:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000800:00000001:1.0:1784949373.366052:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000200:0.0:1784949373.366055:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289693056, offset 224 00000800:00000001:1.0:1784949373.366064:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949373.366073:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:0.0:1784949373.366092:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:0.0:1784949373.366108:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643fb80 00000100:00000001:2.0:1784949373.366109:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949373.366117:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693184 02000000:00000001:2.0:1784949373.366121:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949373.366127:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:0.0:1784949373.366129:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9811 [1] + 224 00000100:00000001:2.0:1784949373.366130:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949373.366139:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949373.366147:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693184 00000020:00000001:2.0:1784949373.366151:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:0.0:1784949373.366152:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1784949373.366154:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:2.0:1784949373.366160:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1784949373.366166:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000400:00000200:0.0:1784949373.366174:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000020:00000001:2.0:1784949373.366174:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:2.0:1784949373.366182:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949373.366190:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1784949373.366191:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949373.366200:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000100:00000200:0.0:1784949373.366206:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949373.366208:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949373.366216:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000020:00000010:2.0:1784949373.366227:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c405a384. 00000020:00000010:2.0:1784949373.366237:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c10b388e. 00000020:00000010:2.0:1784949373.366245:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000521c0956. 00000100:00000040:0.0:1784949373.366247:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949373.366255:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949373.366263:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949373.366267:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949373.366273:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949373.366283:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:2.0:1784949373.366287:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:2.0:1784949373.366288:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949373.366297:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1784949373.366310:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000100:00000001:2.0:1784949373.366311:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949373.366315:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:0.0:1784949373.366323:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949373.366327:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949373.366330:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949373.366334:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.366336:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000040:0.0:1784949373.366348:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00100000:2.0:1784949373.366351:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 815 00010000:00000001:0.0:1784949373.366362:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000040:2.0:1784949373.366363:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000020:00000001:0.0:1784949373.366372:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.366372:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554897792 : -115544154653824 : ffff96e9c7b4bb80) 00000100:00000001:3.0:1784949373.366375:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949373.366385:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1784949373.366388:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:2.0:1784949373.366393:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000000115b4b x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:279/0 lens 224/0 e 0 to 0 dl 1784949384 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784949373.366397:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:0.0:1784949373.366405:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002b339182 x1871651289693056/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:279/0 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1784949373.366406:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1784949373.366414:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000068d5a5a2. 00000100:00000001:2.0:1784949373.366419:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949373.366427:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000023f5e5fc. 00000400:00000200:3.0:1784949373.366429:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000684e85cd 00000100:00000001:2.0:1784949373.366440:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1784949373.366441:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000684e85cd. 00000020:00000010:2.0:1784949373.366454:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000023f5e5fc. 00000100:00000001:3.0:1784949373.366458:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1784949373.366466:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000002b339182 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289693056:12345-0@lo:400: Request processed in 907us (1499us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949373.366469:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000000115b4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289693184:12345-192.168.202.137@tcp:400: 00000100:00000200:3.0:1784949373.366473:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949373.366487:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693184 00000020:00000001:2.0:1784949373.366496:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949373.366503:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949373.366509:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949373.366513:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 839 00000020:00000001:2.0:1784949373.366519:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:2.0:1784949373.366527:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000200:3.0:1784949373.366528:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949373.366529:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000001:2.0:1784949373.366534:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949373.366543:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949373.366544:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949373.366554:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000001:2.0:1784949373.366554:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949373.366562:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949373.366568:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949373.366568:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784949373.366569:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00000001:3.0:1784949373.366577:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949373.366579:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000020:00000010:0.0:1784949373.366586:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000075e36e9f. 00000100:00000001:2.0:1784949373.366589:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:3.0:1784949373.366589:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:2.0:1784949373.366598:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000001:3.0:1784949373.366599:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000020:00000010:0.0:1784949373.366601:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001c9428b0. 02000000:00000010:2.0:1784949373.366605:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 00000000bcfc6676. 00000100:00000001:3.0:1784949373.366608:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:2.0:1784949373.366613:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949373.366616:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949373.366617:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:2.0:1784949373.366619:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949373.366627:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949373.366630:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949373.366634:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000040:2.0:1784949373.366638:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693184 02000000:00000010:0.0:1784949373.366642:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000032060ac0. 00000100:00001000:3.0:1784949373.366642:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=95 v=5 (5 5 5 5) 00010000:00000001:2.0:1784949373.366644:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 02000000:00000001:0.0:1784949373.366656:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00010000:00000200:2.0:1784949373.366657:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000000115b4b x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:279/0 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:3.0:1784949373.366660:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=95 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949373.366665:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949373.366681:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000200:0.0:1784949373.366682:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:2.0:1784949373.366685:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:3.0:1784949373.366691:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949373.366692:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949373.366700:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d30ef19f. 00000100:00001000:2.0:1784949373.366705:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=90 v=5 (1 1 1 1) 00010000:00000001:3.0:1784949373.366706:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1784949373.366715:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949373.366716:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000573e919c. 00000100:00000001:2.0:1784949373.366717:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949373.366727:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949373.366728:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.366734:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:2.0:1784949373.366748:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949373.366750:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.366755:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949373.366756:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949373.366766:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:2.0:1784949373.366776:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ce084714. 00000100:00000200:2.0:1784949373.366787:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289693184, offset 224 00000100:00000001:3.0:1784949373.366790:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949373.366803:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949373.366804:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1784949373.366812:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949373.366826:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949373.366831:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949373.366875:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3944:3944:241:4294967295] 192.168.202.137@tcp LPNI seq info [2523:2523:8:4294967295] 00000100:00100000:3.0:1784949373.366946:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000006b16a92e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289693056:0@lo:400: 00000400:00000200:2.0:1784949373.366962:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:3.0:1784949373.366968:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949373.366979:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000200:2.0:1784949373.366987:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000040:3.0:1784949373.366994:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000006b16a92e x1871651289693056/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1784949373.366997:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f15a9a04. 00000800:00000200:2.0:1784949373.367014:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949373.367032:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000100:00000001:3.0:1784949373.367036:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000200:2.0:1784949373.367039:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f15a9a04 type 1, nob 320 niov 1 nkiov 1 02000000:00000001:3.0:1784949373.367047:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949373.367054:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004565e281. 00000100:00000001:2.0:1784949373.367061:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949373.367067:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:2.0:1784949373.367069:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:2.0:1784949373.367076:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949373.367076:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:2.0:1784949373.367085:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1784949373.367085:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949373.367093:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000020:00000001:3.0:1784949373.367097:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:2.0:1784949373.367104:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000000115b4b x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:279/0 lens 224/224 e 0 to 0 dl 1784949384 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1784949373.367105:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000005f459fde. 02000000:00000001:3.0:1784949373.367120:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949373.367127:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1784949373.367133:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000006b16a92e. 00000100:00100000:2.0:1784949373.367142:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000000115b4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289693184:12345-192.168.202.137@tcp:400: Request processed in 713us (1136us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949373.367146:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1784949373.367153:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949373.367163:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949373.367174:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 815 00000100:00000001:3.0:1784949373.367178:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1784949373.367183:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949373.367186:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949373.367190:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949373.367198:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949373.367199:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1784949373.367209:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949373.367210:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c10b388e. 00000800:00000200:1.0:1784949373.367224:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000010:2.0:1784949373.367224:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000521c0956. 00000800:00000010:1.0:1784949373.367236:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f15a9a04. 00000020:00000010:2.0:1784949373.367238:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c405a384. 00000400:00000200:1.0:1784949373.367250:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1784949373.367252:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:2.0:1784949373.367261:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949373.367268:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:1.0:1784949373.367276:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ce084714 00000400:00000010:1.0:1784949373.367279:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ce084714. 00000100:00000001:1.0:1784949373.367285:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:1.0:1784949373.367288:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949373.367292:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949373.367309:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949373.367313:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949373.367318:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949373.367329:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949373.367341:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643fc00 00000400:00000200:0.0:1784949373.367359:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9819 [1] + 224 00000800:00000001:0.0:1784949373.367366:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949373.367371:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949373.367375:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949373.367383:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:0.0:1784949373.367390:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949373.367397:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949373.367415:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949373.367446:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1784949373.367459:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949373.367465:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949373.367493:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949373.367499:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949373.367505:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949373.367507:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949373.367509:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000400:00000200:1.0:1784949373.367514:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784949373.367516:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784949373.367520:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949373.367526:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949373.367539:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949373.367549:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949373.367551:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949373.367554:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949373.367556:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949373.367558:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949373.367560:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949373.367563:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949373.367570:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=90 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949373.367574:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=90 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949373.367577:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949373.367578:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949373.367581:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949373.367585:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949373.367595:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949373.367598:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949373.367600:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949373.367603:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949373.367604:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949373.367608:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949373.367616:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949373.367619:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949373.367620:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949373.367624:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949373.367632:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000eb4f6eba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289693184:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949373.367637:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949373.367639:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949373.367642:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000eb4f6eba x1871651289693184/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949373.367649:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949373.367651:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949373.367653:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:1.0:1784949373.367657:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949373.367659:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949373.367661:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949373.367663:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949373.367665:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000fc2fde8d. 02000000:00000001:1.0:1784949373.367669:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949373.367670:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949373.367672:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000eb4f6eba. 00000100:00000001:1.0:1784949373.367675:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949373.367677:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949373.367680:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949373.367685:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949373.367686:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949373.367689:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949373.367691:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949373.408055:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306312747, at 4306312748 00000100:00000040:3.0:1784949373.408055:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306312747, at 4306312748 00000100:00000040:0.0:1784949373.408059:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306312747, at 4306312748 00000100:00000010:3.0:1784949373.408075:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002eb614c3. 00000100:00000010:2.0:1784949373.408081:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00000100:00000010:0.0:1784949373.408085:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000078bb23fa. 00000100:00000040:1.0:1784949373.408132:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306312747, at 4306312748 00000100:00000010:1.0:1784949373.408147:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000824e9543. 00000800:00000001:1.0:1784949373.775679:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949373.775713:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949373.775722:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949373.775725:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949374.196035:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949374.196045:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949374.196053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949374.196055:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949374.196152:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949374.196159:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949374.196163:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949374.388054:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949374.388075:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949374.388079:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949374.388088:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949374.388092:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949374.388098:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949374.388103:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949375.220055:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949375.220067:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949375.220079:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949375.220082:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949375.220214:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949375.220221:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949375.220226:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949375.412004:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949375.412017:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949375.412020:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949375.412025:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949375.412029:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949375.412032:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949375.412035:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949376.244036:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949376.244043:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949376.244049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949376.244050:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949376.244106:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949376.244111:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949376.244115:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949376.436022:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949376.436034:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949376.436036:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949376.436041:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949376.436044:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949376.436047:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949376.436049:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949377.268031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949377.268041:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949377.268049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949377.268051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949377.268167:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949377.268176:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949377.268182:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949377.460038:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949377.460056:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949377.460059:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949377.460068:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949377.460072:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949377.460077:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949377.460081:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949377.652055:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.9@tcp 00000800:00000010:3.0:1784949377.652065:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000d74efe59. 00000400:00000200:3.0:1784949377.652079:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949377.652088:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949377.652101:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11650 health :900 00000400:00000200:3.0:1784949377.652110:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949377.652117:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.9@tcp: 1 00000400:00000200:3.0:1784949377.652124:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000800:00000100:3.0:1784949377.652170:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000800:00000010:3.0:1784949377.652175:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f64c9b3a. 00000400:00000200:3.0:1784949377.652182:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949377.652190:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949377.652197:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949377.652204:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000200:1.0:1784949377.652208:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949377.652213:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000200:1.0:1784949377.652233:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x861 00000400:00000001:1.0:1784949377.652244:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949377.652247:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949377.652251:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eb2479c 00000400:00000010:1.0:1784949377.652255:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004eb2479c. 00000400:00000200:1.0:1784949377.652262:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949377.652271:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: -114 00000400:00000010:1.0:1784949377.652276:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f6f633a2. 00000400:00000200:1.0:1784949377.652283:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.9@tcp:-110 00000400:00000200:1.0:1784949377.652286:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 rc -110 00000400:00000200:1.0:1784949377.652290:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.9@tcp 00000400:00000200:1.0:1784949377.652295:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:1.0:1784949377.652297:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949377.652299:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949377.652301:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000002566e96 00000400:00000010:1.0:1784949377.652303:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000002566e96. 00000400:00000200:1.0:1784949377.652305:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949377.652308:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:1.0:1784949377.652311:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000006841197d. 00000400:00000200:1.0:1784949377.652314:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:1.0:1784949377.652317:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:1.0:1784949377.652320:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00010000:00000001:2.0:1784949378.292020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949378.292029:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949378.292037:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949378.292039:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949378.292125:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949378.292131:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949378.292136:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949378.483943:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949378.483956:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949378.483957:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949378.483961:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949378.483964:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949378.483967:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.483969:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949378.483986:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949378.483992:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949378.484000:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a8b02f4c. 00000020:00000040:2.0:1784949378.484006:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949378.484010:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949378.484013:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949378.484017:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949378.484024:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000aba78344. 00000100:00000001:2.0:1784949378.484030:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949378.484038:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949378.484047:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949378.484049:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949378.484053:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8b02f4c] to pc [ptlrpcd_00_01+1] req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949378.484073:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949378.484074:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949378.484077:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949378.484079:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949378.484082:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949378.484084:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b2bae54e. 00000020:00000040:2.0:1784949378.484085:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949378.484087:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949378.484088:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949378.484089:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949378.484093:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000f64c9b3a. 00000100:00000001:2.0:1784949378.484096:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484102:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949378.484103:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1784949378.484107:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1784949378.484111:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949378.484117:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949378.484120:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949378.484121:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949378.484122:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949378.484124:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484125:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949378.484128:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1784949378.484133:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2bae54e] to pc [ptlrpcd_00_02+2] req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949378.484135:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949378.484137:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949378.484143:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484160:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949378.484164:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949378.484167:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1784949378.484169:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949378.484171:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1784949378.484173:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000a8b02f4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289693312:0@lo:400: 00000100:00000001:0.0:1784949378.484177:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949378.484179:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949378.484183:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949378.484186:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949378.484187:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11655) 02000000:00000001:0.0:1784949378.484189:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949378.484191:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949378.484194:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:0.0:1784949378.484195:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000001c9428b0. 00000100:00000001:1.0:1784949378.484197:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1784949378.484199:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949378.484204:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1784949378.484206:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000055895529. 00000100:00000001:3.0:1784949378.484211:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1784949378.484216:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000100:00000001:1.0:1784949378.484217:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949378.484218:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484221:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949378.484225:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000200:0.0:1784949378.484226:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693312, portal 25 02000000:00000001:1.0:1784949378.484228:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949378.484230:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949378.484233:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469582720 : -115544239968896 : ffff96e9c29eed80) 02000000:00000001:1.0:1784949378.484235:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949378.484239:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949378.484244:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b2bae54e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289693440:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949378.484252:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949378.484253:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949378.484258:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 02000000:00000001:1.0:1784949378.484260:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000010:0.0:1784949378.484261:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 02000000:00000001:1.0:1784949378.484264:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949378.484265:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289693312, offset 0 02000000:00000001:1.0:1784949378.484271:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1784949378.484275:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:1.0:1784949378.484279:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000005ae2686. 02000000:00000001:1.0:1784949378.484283:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949378.484288:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000010:1.0:1784949378.484289:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000400:00000010:1.0:1784949378.484297:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000200:0.0:1784949378.484301:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643fc80 00000100:00000200:1.0:1784949378.484304:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693440, portal 10 00000400:00000200:0.0:1784949378.484309:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8e9d [2] + 0 00000100:00000001:1.0:1784949378.484310:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949378.484314:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469586304 : -115544239965312 : ffff96e9c29efb80) 00000400:00000010:0.0:1784949378.484315:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e8bdea01. 00000100:00000040:1.0:1784949378.484326:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949378.484327:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000059548475 00000400:00000200:0.0:1784949378.484338:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949378.484338:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949378.484344:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949378.484348:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000010:1.0:1784949378.484351:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030a9ed2a. 00000400:00000200:0.0:1784949378.484355:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000059548475 00000400:00000010:0.0:1784949378.484360:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000059548475. 00000100:00000001:0.0:1784949378.484366:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949378.484367:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289693440, offset 0 00000100:00000200:0.0:1784949378.484369:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:0.0:1784949378.484375:0:337119:0:(events.c:337:request_in_callback()) incoming req@00000000e164874b x1871651289693312 msgsize 224 00000400:00000200:1.0:1784949378.484379:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784949378.484384:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:0.0:1784949378.484387:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:1.0:1784949378.484395:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:0.0:1784949378.484398:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:1.0:1784949378.484402:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949378.484409:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949378.484414:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949378.484418:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949378.484422:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3945:3945:243:4294967295] 192.168.202.137@tcp LPNI seq info [2524:2524:8:4294967295] 00000100:00000001:2.0:1784949378.484433:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:1.0:1784949378.484437:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00100000:2.0:1784949378.484439:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693312 02000000:00000001:2.0:1784949378.484445:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:2.0:1784949378.484449:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000800:00000200:1.0:1784949378.484450:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949378.484451:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000100:00000001:2.0:1784949378.484454:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949378.484455:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000800:00000010:1.0:1784949378.484458:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000000475ac2. 00000100:00000001:0.0:1784949378.484461:0:337119:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1784949378.484463:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949378.484468:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949378.484468:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00100000:2.0:1784949378.484470:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693312 00000020:00000001:2.0:1784949378.484476:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949378.484481:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000800:00000200:1.0:1784949378.484482:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:0.0:1784949378.484483:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949378.484487:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.0:1784949378.484488:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000000475ac2 type 1, nob 320 niov 1 nkiov 1 00000100:00000040:0.0:1784949378.484489:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1784949378.484494:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784949378.484501:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:0.0:1784949378.484505:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949378.484507:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949378.484509:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484510:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1784949378.484515:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949378.484516:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2262 usecs (3 jiffies) 00000100:00000001:0.0:1784949378.484519:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484521:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306317827 00000020:00000010:2.0:1784949378.484524:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000008aed2f1. 00000100:00000001:0.0:1784949378.484526:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306317866 00000100:00000010:1.0:1784949378.484527:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000001:2.0:1784949378.484531:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949378.484536:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000008aed2f1. 00000100:00000001:1.0:1784949378.484538:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949378.484545:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306317827 00000020:00000010:2.0:1784949378.484546:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c405a384. 00000100:00000010:1.0:1784949378.484549:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:0.0:1784949378.484550:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306317827 00000020:00000010:2.0:1784949378.484571:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000003e303dd0. 00000100:00000001:1.0:1784949378.484572:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949378.484579:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306317866 00000100:00000001:1.0:1784949378.484583:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306317827 00000020:00000010:2.0:1784949378.484585:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000521c0956. 00000100:00000001:0.0:1784949378.484588:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306317827 00000100:00000010:1.0:1784949378.484591:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f595b698. 00000100:00000001:1.0:1784949378.484600:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000100:00000001:0.0:1784949378.484602:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306317866 00000100:00000001:1.0:1784949378.484608:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306317827 00000100:00000040:2.0:1784949378.484610:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949378.484612:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306317827 00000100:00000010:1.0:1784949378.484616:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000300e92c9. 00000100:00000001:2.0:1784949378.484621:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949378.484625:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306317866 00000100:00000001:1.0:1784949378.484625:0:337121:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000001:2.0:1784949378.484626:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949378.484633:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484639:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949378.484642:0:337119:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306317827 00000100:00000001:2.0:1784949378.484645:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484648:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484653:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949378.484656:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484658:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484661:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949378.484665:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484671:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484671:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949378.484676:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949378.484681:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949378.484681:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.484687:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949378.484693:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949378.484696:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:1.0:1784949378.484698:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949378.484706:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1784949378.484707:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949378.484713:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000800:00000001:0.0:1784949378.484714:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949378.484718:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000000475ac2. 00000800:00000001:0.0:1784949378.484721:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784949378.484749:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 840 00000800:00000001:0.0:1784949378.484751:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949378.484753:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:2.0:1784949378.484754:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:2.0:1784949378.484759:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782158928 : -115538927392688 : ffff96eaff466850) 00000400:00000200:0.0:1784949378.484761:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:2.0:1784949378.484768:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e164874b x1871651289693312/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:284/0 lens 224/0 e 0 to 0 dl 1784949389 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949378.484768:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:0.0:1784949378.484776:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643fd00 00000400:00000200:3.0:1784949378.484780:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030a9ed2a 00000400:00000010:3.0:1784949378.484786:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000030a9ed2a. 00000100:00000001:2.0:1784949378.484787:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:2.0:1784949378.484791:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000008aed2f1. 00000100:00000001:3.0:1784949378.484793:0:337113:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1784949378.484795:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949378.484799:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 183824 00000020:00000010:2.0:1784949378.484800:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000008aed2f1. 00000100:00000200:3.0:1784949378.484802:0:337113:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949378.484809:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00100000:2.0:1784949378.484809:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e164874b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289693312:12345-0@lo:400: 00000100:00000200:2.0:1784949378.484830:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693312 00000800:00000001:0.0:1784949378.484831:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949378.484834:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000400:00000200:0.0:1784949378.484835:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949378.484838:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000400:00000200:0.0:1784949378.484841:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:2.0:1784949378.484842:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.484847:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000020:00000001:2.0:1784949378.484848:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000200:0.0:1784949378.484851:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000020:00000001:2.0:1784949378.484852:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784949378.484868:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000010:0.0:1784949378.484869:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f84ecbdd. 00000100:00000001:3.0:1784949378.484869:0:337113:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1784949378.484871:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949378.484875:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000f84ecbdd x1871651289693440 msgsize 224 00000020:00000001:2.0:1784949378.484876:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:3.0:1784949378.484878:0:337113:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1784949378.484879:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:2.0:1784949378.484883:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949378.484888:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00100000:0.0:1784949378.484889:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:2.0:1784949378.484893:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949378.484897:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:2.0:1784949378.484902:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000008142e1f1. 00000100:00000001:0.0:1784949378.484926:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1784949378.484934:0:337113:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949378.484936:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949378.484937:0:337113:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1784949378.484939:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000800:00000001:0.0:1784949378.484941:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949378.484943:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693440 02000000:00000001:1.0:1784949378.484947:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 02000000:00000001:2.0:1784949378.484947:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484951:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949378.484952:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.484954:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949378.484955:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949378.484959:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949378.484960:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693312 00000100:00100000:1.0:1784949378.484964:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693440 00010000:00000001:2.0:1784949378.484965:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000020:00000001:1.0:1784949378.484969:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949378.484971:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00010000:00000200:2.0:1784949378.484972:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e164874b x1871651289693312/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:284/0 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1784949378.484973:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949378.484976:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949378.484982:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:1.0:1784949378.484987:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00010000:00000001:2.0:1784949378.484989:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949378.484990:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00010000:00000001:2.0:1784949378.484992:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949378.484995:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00001000:2.0:1784949378.484996:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=100 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949378.485000:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949378.485002:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:1.0:1784949378.485003:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00000040:2.0:1784949378.485007:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000020:00000010:1.0:1784949378.485028:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006841197d. 00000100:00000001:2.0:1784949378.485028:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000020:00000010:1.0:1784949378.485036:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004db98700. 02000000:00000001:2.0:1784949378.485038:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949378.485043:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949378.485046:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:2.0:1784949378.485049:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949378.485054:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000040:1.0:1784949378.485057:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000400:00000010:2.0:1784949378.485061:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000100:00000001:1.0:1784949378.485062:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949378.485067:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000200:2.0:1784949378.485070:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289693312, offset 224 00000100:00000001:1.0:1784949378.485072:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949378.485080:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000001:1.0:1784949378.485081:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949378.485086:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:1.0:1784949378.485088:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949378.485096:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949378.485104:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643fc80 00000100:00000001:1.0:1784949378.485108:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949378.485115:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x982d [1] + 224 00000400:00000200:2.0:1784949378.485123:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949378.485125:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949378.485131:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000200:2.0:1784949378.485132:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949378.485142:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1784949378.485143:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 816 00000100:00000040:1.0:1784949378.485149:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000200:2.0:1784949378.485152:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949378.485153:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469560064 : -115544239991552 : ffff96e9c29e9500) 00000100:00000040:1.0:1784949378.485167:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000f84ecbdd x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:284/0 lens 224/0 e 0 to 0 dl 1784949389 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949378.485182:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949378.485187:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949378.485193:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000fd72bb58. 00000100:00000001:1.0:1784949378.485199:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1784949378.485203:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000fd72bb58. 00000100:00100000:1.0:1784949378.485208:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000f84ecbdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289693440:12345-192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949378.485212:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693440 00000100:00000001:2.0:1784949378.485213:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1784949378.485215:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:1.0:1784949378.485221:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949378.485223:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949378.485225:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000020:00000001:1.0:1784949378.485227:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000400:00000010:2.0:1784949378.485230:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000020:00000001:1.0:1784949378.485231:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000001:2.0:1784949378.485237:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000020:00000001:1.0:1784949378.485238:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:2.0:1784949378.485243:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000020:00000001:1.0:1784949378.485244:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949378.485249:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:2.0:1784949378.485250:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949378.485252:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949378.485257:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.485259:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949378.485261:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000020:00000001:1.0:1784949378.485264:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949378.485266:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1784949378.485267:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:1.0:1784949378.485271:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000020:00000001:2.0:1784949378.485273:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949378.485276:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000020:00000001:2.0:1784949378.485278:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000001:0.0:1784949378.485279:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:1.0:1784949378.485282:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000003e4182b6. 00000400:00000001:0.0:1784949378.485285:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1784949378.485289:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e164874b x1871651289693312/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:284/0 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1784949378.485290:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949378.485291:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000055895529. 00000100:00000001:1.0:1784949378.485298:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949378.485302:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000020:00000001:1.0:1784949378.485306:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949378.485309:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00010000:00000040:1.0:1784949378.485314:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693440 00000100:00000001:0.0:1784949378.485318:0:337119:0:(events.c:78:reply_in_callback()) Process entered 00010000:00000001:1.0:1784949378.485323:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000200:0.0:1784949378.485333:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000200:1.0:1784949378.485335:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000f84ecbdd x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:284/0 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949378.485336:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e164874b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289693312:12345-0@lo:400: Request processed in 539us (951us total) trans 0 rc 0/0 00000100:00100000:2.0:1784949378.485365:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 840 00000100:00000040:2.0:1784949378.485376:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:2.0:1784949378.485383:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1784949378.485387:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00010000:00000001:1.0:1784949378.485392:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000020:00000010:2.0:1784949378.485397:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000003e303dd0. 00010000:00000001:1.0:1784949378.485398:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949378.485400:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00001000:1.0:1784949378.485407:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=95 v=5 (1 1 1 1) 00000020:00000010:2.0:1784949378.485408:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000521c0956. 00000020:00000010:2.0:1784949378.485417:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c405a384. 00000100:00000001:1.0:1784949378.485423:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:2.0:1784949378.485428:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000040:1.0:1784949378.485431:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949378.485433:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1784949378.485435:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:0.0:1784949378.485439:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.485440:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000010:2.0:1784949378.485441:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000008142e1f1. 00000100:00000001:0.0:1784949378.485445:0:337119:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949378.485451:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949378.485453:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 02000000:00000001:2.0:1784949378.485453:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 02000000:00000001:1.0:1784949378.485456:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.485459:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949378.485460:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.485462:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1784949378.485464:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949378.485469:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949378.485471:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000200:2.0:1784949378.485471:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00001000:0.0:1784949378.485476:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=100 v=5 (5 5 5 5) 00000400:00000010:1.0:1784949378.485480:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a6647b6e. 00000400:00000010:2.0:1784949378.485482:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000068d5a5a2. 00000100:00001000:0.0:1784949378.485484:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=100 v=5 (1 1 1 1) 00000100:00000200:1.0:1784949378.485487:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289693440, offset 224 00000400:00000010:2.0:1784949378.485494:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000097d07cda. 00000100:00000001:0.0:1784949378.485499:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949378.485502:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949378.485509:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00010000:00000001:0.0:1784949378.485510:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1784949378.485515:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.485519:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949378.485525:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000100:00000040:0.0:1784949378.485527:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949378.485539:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3946:3946:243:4294967295] 192.168.202.137@tcp LPNI seq info [2525:2525:8:4294967295] 00000100:00000001:0.0:1784949378.485549:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949378.485555:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949378.485560:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949378.485567:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000040:0.0:1784949378.485571:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949378.485587:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00100000:0.0:1784949378.485596:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000a8b02f4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289693312:0@lo:400: 00000800:00000010:1.0:1784949378.485598:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000094ed938f. 00000100:00000001:0.0:1784949378.485609:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1784949378.485611:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949378.485614:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949378.485620:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000a8b02f4c x1871651289693312/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1784949378.485626:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949378.485634:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000094ed938f type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949378.485645:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949378.485648:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949378.485651:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000001c9428b0. 02000000:00000001:0.0:1784949378.485656:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949378.485658:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1784949378.485659:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1784949378.485661:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949378.485668:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:1.0:1784949378.485668:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 02000000:00000010:0.0:1784949378.485673:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000aba78344. 00010000:00000001:1.0:1784949378.485675:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949378.485680:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949378.485682:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:1.0:1784949378.485684:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 02000000:00000001:0.0:1784949378.485687:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949378.485690:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000a8b02f4c. 00000100:00000040:1.0:1784949378.485695:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f84ecbdd x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:284/0 lens 224/224 e 0 to 0 dl 1784949389 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949378.485697:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949378.485702:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949378.485708:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949378.485733:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1784949378.485733:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000f84ecbdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289693440:12345-192.168.202.137@tcp:400: Request processed in 545us (856us total) trans 0 rc 0/0 00000100:00000001:0.0:1784949378.485736:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949378.485742:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949378.485744:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949378.485745:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 816 00000100:00000040:1.0:1784949378.485750:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:1.0:1784949378.485752:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:1.0:1784949378.485753:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:1.0:1784949378.485756:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004db98700. 00000020:00000010:1.0:1784949378.485759:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00000020:00000010:1.0:1784949378.485762:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006841197d. 00000020:00000040:1.0:1784949378.485764:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:1.0:1784949378.485766:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949378.485798:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949378.485826:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000094ed938f. 00000400:00000200:2.0:1784949378.485832:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949378.485833:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949378.485839:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949378.485841:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949378.485843:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a6647b6e 00000800:00000001:3.0:1784949378.485845:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784949378.485847:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a6647b6e. 00000800:00000001:3.0:1784949378.485849:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949378.485852:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949378.485855:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949378.485858:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949378.485877:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643fd00 00000400:00000200:3.0:1784949378.485882:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9835 [1] + 224 00000800:00000001:3.0:1784949378.485886:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949378.485892:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949378.485894:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949378.485897:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949378.485900:0:337113:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949378.485902:0:337113:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949378.485929:0:337113:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949378.485952:0:337113:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949378.485957:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949378.485959:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.485983:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949378.485986:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949378.485989:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949378.485990:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949378.485991:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000400:00000200:1.0:1784949378.485994:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784949378.486012:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784949378.486015:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949378.486021:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949378.486030:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949378.486047:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949378.486049:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.486051:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949378.486053:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949378.486054:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949378.486056:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949378.486057:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949378.486062:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=95 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949378.486065:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=95 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949378.486068:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949378.486069:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949378.486071:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949378.486075:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949378.486083:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.486085:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949378.486087:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949378.486089:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949378.486090:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949378.486093:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949378.486098:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949378.486100:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949378.486102:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949378.486104:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949378.486140:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b2bae54e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289693440:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949378.486144:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.486145:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949378.486147:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b2bae54e x1871651289693440/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949378.486160:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949378.486162:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949378.486164:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000005ae2686. 02000000:00000001:1.0:1784949378.486168:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949378.486170:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949378.486171:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949378.486173:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949378.486175:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000f64c9b3a. 02000000:00000001:1.0:1784949378.486183:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949378.486185:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949378.486186:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b2bae54e. 00000100:00000001:1.0:1784949378.486189:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949378.486191:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.486193:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949378.486197:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949378.486198:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949378.486200:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949378.486202:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949378.526970:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306317866, at 4306317867 00000100:00000040:1.0:1784949378.526980:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306317866, at 4306317867 00000100:00000010:3.0:1784949378.526983:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000300e92c9. 00000100:00000040:0.0:1784949378.526990:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306317866, at 4306317867 00000100:00000010:1.0:1784949378.526991:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:0.0:1784949378.527001:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000040:2.0:1784949378.527002:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306317866, at 4306317867 00000100:00000010:2.0:1784949378.527014:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f595b698. 00000800:00000100:3.0:1784949378.675999:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.13@tcp 00000800:00000010:3.0:1784949378.676004:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000009662d9a. 00000400:00000200:3.0:1784949378.676012:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949378.676019:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949378.676026:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949378.676031:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.13@tcp: 1 00000400:00000200:3.0:1784949378.676037:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000200:2.0:1784949378.676064:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:2.0:1784949378.676073:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000005d112158. 00000400:00000010:2.0:1784949378.676081:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000200:1.0:1784949378.676084:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:2.0:1784949378.676088:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001deb77ec. 00000400:00000200:1.0:1784949378.676095:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x861 00000400:00000010:2.0:1784949378.676100:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000afc1f2c6. 00000400:00000001:1.0:1784949378.676103:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949378.676107:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949378.676109:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3f5c273 00000400:00000010:1.0:1784949378.676111:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d3f5c273. 00000400:00000200:1.0:1784949378.676116:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949378.676120:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: -114 00000400:00000010:1.0:1784949378.676125:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f0e4e38. 00000400:00000200:2.0:1784949378.676126:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:1.0:1784949378.676134:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.13@tcp:-110 00000400:00000200:1.0:1784949378.676138:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 rc -110 00000400:00000200:2.0:1784949378.676147:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949378.676157:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949378.676162:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3947:3947:244:4294967295] 192.168.202.137@tcp LPNI seq info [2526:2526:8:4294967295] 00000400:00000200:2.0:1784949378.676169:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:2.0:1784949378.676177:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000400:00000200:1.0:1784949378.676186:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.13@tcp 00000800:00000200:2.0:1784949378.676192:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949378.676199:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f15a9a04. 00000800:00000200:2.0:1784949378.676209:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949378.676220:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784949378.676225:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f15a9a04 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949378.676247:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11652 health :900 00000800:00000200:0.0:1784949378.676400:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949378.676407:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f15a9a04. 00000400:00000200:0.0:1784949378.676414:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949378.676422:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949378.676428:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:2.0:1784949378.676431:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949378.676433:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949378.676448:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949378.676454:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949378.676476:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949378.676489:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949378.676507:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949378.676519:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949378.676529:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949378.676534:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3948:3948:244:4294967295] 192.168.202.137@tcp LPNI seq info [2527:2527:8:4294967295] 00000400:00000200:2.0:1784949378.676544:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949378.676551:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949378.676555:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000284ae7c9. 00000800:00000200:2.0:1784949378.676560:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949378.676565:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949378.676568:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000284ae7c9 type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949378.676679:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949378.676682:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949378.676689:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000284ae7c9. 00000800:00000001:3.0:1784949378.676690:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949378.676695:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949378.676698:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949378.676705:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949378.676709:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949378.676715:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949378.676721:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949378.676731:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949378.676748:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9849 00000800:00000001:3.0:1784949378.676755:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949378.676764:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949378.676767:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949378.676771:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949378.676779:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001deb77ec 00000400:00000010:3.0:1784949378.676782:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000001deb77ec. 00000400:00000200:3.0:1784949378.676788:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949378.676793:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000005d112158. 00000400:00000010:3.0:1784949378.676798:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000800:00000001:3.0:1784949378.676840:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949378.676844:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949379.316064:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949379.316076:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949379.316088:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949379.316092:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949379.316245:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949379.316251:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949379.316256:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949379.508034:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949379.508051:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949379.508053:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949379.508060:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949379.508064:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949379.508069:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949379.508072:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949379.699997:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000800:00000010:3.0:1784949379.700004:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000004c284093. 00000400:00000200:3.0:1784949379.700015:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949379.700022:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949379.700027:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11652 health :900 00000400:00000200:3.0:1784949379.700033:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949379.700036:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000200:3.0:1784949379.700040:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:1.0:1784949379.700091:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949379.700100:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:1.0:1784949379.700107:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949379.700109:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949379.700112:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e6156599 00000400:00000010:1.0:1784949379.700114:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e6156599. 00000400:00000200:1.0:1784949379.700119:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949379.700122:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:1.0:1784949379.700126:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b1cbf66f. 00000400:00000200:1.0:1784949379.700131:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:1.0:1784949379.700133:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:1.0:1784949379.700136:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00010000:00000001:2.0:1784949380.340064:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949380.340077:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949380.340089:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949380.340092:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949380.340174:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949380.340181:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949380.340185:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949380.532120:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949380.532150:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949380.532154:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949380.532163:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949380.532168:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949380.532173:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949380.532178:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949380.724036:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:2.0:1784949380.724050:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 000000006940b099. 00000400:00000010:2.0:1784949380.724067:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000010:2.0:1784949380.724075:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000400:00000010:2.0:1784949380.724087:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005c9e133d. 00000400:00000200:2.0:1784949380.724094:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949380.724113:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949380.724123:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949380.724128:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3949:3949:245:4294967295] 192.168.202.137@tcp LPNI seq info [2528:2528:8:4294967295] 00000400:00000200:2.0:1784949380.724137:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:2.0:1784949380.724147:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:2.0:1784949380.724154:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949380.724161:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000284ae7c9. 00000800:00000200:2.0:1784949380.724170:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949380.724181:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784949380.724187:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000284ae7c9 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949380.724212:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11654 health :900 00000800:00000200:0.0:1784949380.724440:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949380.724448:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000284ae7c9. 00000400:00000200:0.0:1784949380.724461:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949380.724466:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949380.724474:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:0.0:1784949380.724482:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000800:00000001:2.0:1784949380.724487:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949380.724488:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949380.724498:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949380.724508:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949380.724518:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949380.724536:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949380.724549:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949380.724561:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949380.724566:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3950:3950:245:4294967295] 192.168.202.137@tcp LPNI seq info [2529:2529:8:4294967295] 00000400:00000200:2.0:1784949380.724577:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949380.724582:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949380.724586:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003cb28a6d. 00000800:00000200:2.0:1784949380.724592:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949380.724597:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949380.724600:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003cb28a6d type 3, nob 144 niov 1 nkiov 1 00000800:00000001:3.0:1784949380.724663:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949380.724734:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949380.724741:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003cb28a6d. 00000400:00000200:2.0:1784949380.724748:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949380.724751:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949380.724758:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949380.724767:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949380.724773:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949380.724776:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949380.724782:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949380.724793:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949380.724832:0:337112:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x984d 00000800:00000001:0.0:1784949380.724840:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949380.724849:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949380.724852:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949380.724857:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949380.724864:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 00000400:00000010:0.0:1784949380.724867:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 00000400:00000200:0.0:1784949380.724871:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949380.724874:0:337112:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 000000006940b099. 00000400:00000010:0.0:1784949380.724878:0:337112:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000800:00000001:0.0:1784949380.724887:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949380.724890:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949381.364015:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949381.364024:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949381.364033:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949381.364036:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949381.364141:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949381.364148:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949381.364153:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949381.556013:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949381.556030:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949381.556032:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949381.556039:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949381.556042:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949381.556046:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949381.556050:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949381.748045:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.3@tcp 00000800:00000010:3.0:1784949381.748054:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e79c6895. 00000400:00000200:3.0:1784949381.748066:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949381.748073:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949381.748080:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11654 health :900 00000400:00000200:3.0:1784949381.748087:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949381.748092:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.3@tcp: 1 00000400:00000200:3.0:1784949381.748097:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000200:1.0:1784949381.748186:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949381.748203:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x861 00000400:00000001:1.0:1784949381.748214:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949381.748218:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949381.748239:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:1.0:1784949381.748244:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:1.0:1784949381.748251:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949381.748257:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: -114 00000400:00000010:1.0:1784949381.748263:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032fe57fd. 00000400:00000200:1.0:1784949381.748277:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.3@tcp:-110 00000400:00000200:1.0:1784949381.748281:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 rc -110 00000400:00000200:1.0:1784949381.748287:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.3@tcp 00010000:00000001:2.0:1784949382.388030:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949382.388040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949382.388048:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949382.388052:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949382.388055:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949382.388059:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949382.388063:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949382.388066:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949382.388069:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949382.388071:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949382.388164:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949382.388173:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949382.388179:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949382.580082:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949382.580099:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949382.580102:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949382.580110:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949382.580114:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949382.580118:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949382.580123:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949382.772013:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.7@tcp 00000800:00000010:3.0:1784949382.772019:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000030b7fbec. 00000400:00000200:2.0:1784949382.772047:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:3.0:1784949382.772048:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949382.772060:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:2.0:1784949382.772061:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000a78d6a97. 00000400:00000200:3.0:1784949382.772071:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000010:2.0:1784949382.772076:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000200:3.0:1784949382.772081:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.7@tcp: 1 00000400:00000010:2.0:1784949382.772086:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000200:3.0:1784949382.772092:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:2.0:1784949382.772109:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a33a2170. 00000400:00000200:2.0:1784949382.772121:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949382.772138:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949382.772146:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949382.772153:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3951:3951:247:4294967295] 192.168.202.137@tcp LPNI seq info [2530:2530:8:4294967295] 00000400:00000200:2.0:1784949382.772163:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:2.0:1784949382.772173:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000400:00000200:1.0:1784949382.772184:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:2.0:1784949382.772185:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949382.772198:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000003cb28a6d. 00000400:00000200:1.0:1784949382.772199:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x861 00000400:00000001:1.0:1784949382.772209:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949382.772213:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:2.0:1784949382.772217:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949382.772219:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:1.0:1784949382.772227:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:1.0:1784949382.772233:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000800:00000200:2.0:1784949382.772239:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:1.0:1784949382.772240:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: -114 00000400:00000010:1.0:1784949382.772248:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f11303f8. 00000800:00000200:2.0:1784949382.772253:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000003cb28a6d type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949382.772257:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.7@tcp:-110 00000400:00000200:1.0:1784949382.772264:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 rc -110 00000400:00000200:1.0:1784949382.772270:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.7@tcp 00000400:00000200:2.0:1784949382.772290:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11656 health :800 00000800:00000200:3.0:1784949382.772437:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949382.772442:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000003cb28a6d. 00000400:00000200:3.0:1784949382.772449:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949382.772458:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000800:00000001:1.0:1784949382.772461:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949382.772463:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949382.772469:0:337114:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:1.0:1784949382.772480:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1784949382.772485:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949382.772493:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949382.772502:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:1.0:1784949382.772517:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1784949382.772527:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1784949382.772537:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949382.772542:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3952:3952:247:4294967295] 192.168.202.137@tcp LPNI seq info [2531:2531:8:4294967295] 00000400:00000200:1.0:1784949382.772552:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:1.0:1784949382.772556:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949382.772561:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f495c80e. 00000800:00000200:1.0:1784949382.772566:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949382.772571:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:1.0:1784949382.772574:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f495c80e type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949382.772691:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1784949382.772721:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949382.772723:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:1.0:1784949382.772730:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f495c80e. 00000800:00000001:0.0:1784949382.772735:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949382.772736:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949382.772737:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949382.772746:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:1.0:1784949382.772749:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949382.772755:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949382.772767:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949382.772805:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9851 00000800:00000001:3.0:1784949382.772813:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949382.772826:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949382.772830:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949382.772835:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949382.772840:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:3.0:1784949382.772844:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:3.0:1784949382.772848:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949382.772852:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000a78d6a97. 00000400:00000010:3.0:1784949382.772856:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000800:00000001:3.0:1784949382.772865:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949382.772869:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949383.412020:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949383.412031:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949383.412039:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949383.412041:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949383.412171:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949383.412178:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949383.412182:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949383.604041:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000040:2.0:1784949383.604054:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949383.604061:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949383.604065:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1784949383.604070:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949383.604078:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:2.0:1784949383.604079:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000000b40f51. 00000100:00000001:0.0:1784949383.604083:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949383.604090:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1784949383.604092:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949383.604096:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949383.604101:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949383.604107:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949383.604114:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949383.604129:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000003cb28a6d. 00000100:00000001:2.0:1784949383.604141:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949383.604158:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949383.604179:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949383.604184:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949383.604193:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000000b40f51] to pc [ptlrpcd_00_03+3] req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949383.604244:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949383.604249:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:2.0:1784949383.604256:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:2.0:1784949383.604261:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949383.604268:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1784949383.604274:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c646352a. 00000020:00000040:2.0:1784949383.604279:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949383.604283:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1784949383.604286:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1784949383.604291:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949383.604297:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000284ae7c9. 00000100:00000001:2.0:1784949383.604303:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949383.604315:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1784949383.604332:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949383.604335:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949383.604338:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949383.604346:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c646352a] to pc [ptlrpcd_00_00+0] req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949383.604348:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949383.604355:0:337122:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1784949383.604363:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1784949383.604369:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949383.604373:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949383.604393:0:337122:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949383.604396:0:337122:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949383.604402:0:337122:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1784949383.604406:0:337122:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949383.604408:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949383.604412:0:337122:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949383.604420:0:337122:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@0000000000b40f51 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289693568:0@lo:400: 00000100:00000001:1.0:1784949383.604424:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949383.604424:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11660) 00000100:00000001:3.0:1784949383.604429:0:337122:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1784949383.604433:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1784949383.604438:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949383.604442:0:337122:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1784949383.604447:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1784949383.604448:0:337122:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.604455:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949383.604457:0:337122:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1784949383.604465:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949383.604470:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1784949383.604471:0:337122:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000035b54a67. 00000100:00000001:1.0:1784949383.604474:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1784949383.604482:0:337122:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949383.604489:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949383.604499:0:337122:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001004a12d. 00000100:00000001:1.0:1784949383.604512:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1784949383.604513:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949383.604516:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949383.604519:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bb55de7f. 00000100:00000001:0.0:1784949383.604522:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949383.604523:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949383.604531:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949383.604536:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949383.604538:0:337122:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693568, portal 25 00000100:00100000:1.0:1784949383.604541:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000c646352a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289693696:192.168.202.137@tcp:400: 00000100:00000001:3.0:1784949383.604550:0:337122:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949383.604558:0:337122:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469581824 : -115544239969792 : ffff96e9c29eea00) 00000100:00000001:1.0:1784949383.604560:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949383.604566:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949383.604571:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949383.604575:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1784949383.604577:0:337122:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784949383.604584:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000006841197d. 02000000:00000001:1.0:1784949383.604593:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949383.604605:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000001:3.0:1784949383.604611:0:337122:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000010:1.0:1784949383.604616:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000100:00000040:3.0:1784949383.604625:0:337122:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000100:00000200:1.0:1784949383.604627:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693696, portal 10 00000100:00000001:1.0:1784949383.604634:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:3.0:1784949383.604635:0:337122:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000100:00000001:1.0:1784949383.604636:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469579136 : -115544239972480 : ffff96e9c29edf80) 00000100:00000200:3.0:1784949383.604649:0:337122:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289693568, offset 0 00000100:00000040:1.0:1784949383.604653:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949383.604671:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949383.604673:0:337122:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:1.0:1784949383.604683:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949383.604690:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000100:00000200:1.0:1784949383.604695:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289693696, offset 0 00000400:00000200:1.0:1784949383.604707:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949383.604710:0:337122:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:1.0:1784949383.604726:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949383.604733:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949383.604739:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949383.604744:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949383.604748:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000400:00000200:1.0:1784949383.604752:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3953:3953:247:4294967295] 192.168.202.137@tcp LPNI seq info [2532:2532:8:4294967295] 00000400:00000200:1.0:1784949383.604768:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949383.604803:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949383.604810:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f495c80e. 00000800:00000200:1.0:1784949383.604819:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:1.0:1784949383.604830:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:1.0:1784949383.604836:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f495c80e type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1784949383.604845:0:337122:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643fd80 00000400:00000200:3.0:1784949383.604861:0:337122:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8ebd [2] + 0 00000100:00000001:1.0:1784949383.604865:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949383.604870:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000005a5d8678. 00000100:00000040:1.0:1784949383.604876:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2262 usecs (3 jiffies) 00000100:00000001:1.0:1784949383.604885:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306322947 00000400:00000200:3.0:1784949383.604886:0:337122:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000bc3ccb92 00000100:00000010:1.0:1784949383.604896:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f595b698. 00000100:00000001:1.0:1784949383.604954:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 0 00000100:00000001:1.0:1784949383.604963:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306322948 00000100:00000010:1.0:1784949383.604970:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000300e92c9. 00000400:00000200:3.0:1784949383.604971:0:337122:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949383.604978:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949383.604989:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306322948 00000400:00000200:3.0:1784949383.604990:0:337122:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000010:1.0:1784949383.605000:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000400:00000200:3.0:1784949383.605006:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bc3ccb92 00000100:00000001:1.0:1784949383.605007:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 2 00000400:00000010:3.0:1784949383.605017:0:337122:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 00000000bc3ccb92. 00000100:00000001:1.0:1784949383.605025:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306322948 00000100:00000001:3.0:1784949383.605031:0:337122:0:(events.c:286:request_in_callback()) Process entered 00000100:00000010:1.0:1784949383.605033:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000200:3.0:1784949383.605039:0:337122:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784949383.605042:0:337120:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request now active for cpu 3 00000100:00000040:3.0:1784949383.605050:0:337122:0:(events.c:337:request_in_callback()) incoming req@000000000ae6b462 x1871651289693568 msgsize 224 00000100:00000001:1.0:1784949383.605052:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949383.605065:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.605070:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949383.605079:0:337122:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:1.0:1784949383.605081:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949383.605088:0:337122:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000800:00000200:0.0:1784949383.605098:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949383.605105:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f495c80e. 00000800:00000001:2.0:1784949383.605110:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949383.605116:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949383.605116:0:337122:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000200:0.0:1784949383.605130:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:2.0:1784949383.605134:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949383.605136:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0e1f1a0 00000400:00000010:3.0:1784949383.605144:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0e1f1a0. 00000800:00000001:2.0:1784949383.605148:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949383.605151:0:337122:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1784949383.605153:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a01227d5 00000400:00000010:0.0:1784949383.605160:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a01227d5. 00000800:00000001:2.0:1784949383.605160:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949383.605162:0:337122:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949383.605174:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1784949383.605190:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949383.605195:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949383.605197:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949383.605207:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949383.605234:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643fe00 00000100:00000001:0.0:1784949383.605242:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949383.605254:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949383.605255:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949383.605267:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 184048 00000100:00000001:3.0:1784949383.605267:0:337122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949383.605281:0:337122:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1784949383.605289:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:0.0:1784949383.605295:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949383.605296:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306322987 00000100:00000001:0.0:1784949383.605301:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1784949383.605304:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949383.605311:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949383.605319:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4306322947 00000400:00000200:2.0:1784949383.605320:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949383.605333:0:337113:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1784949383.605339:0:337113:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784949383.605339:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306322987 00000100:00000001:0.0:1784949383.605344:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000010:2.0:1784949383.605348:0:337113:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003901a72b. 00000100:00100000:0.0:1784949383.605352:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693568 00000100:00000040:2.0:1784949383.605356:0:337113:0:(events.c:337:request_in_callback()) incoming req@000000003901a72b x1871651289693696 msgsize 224 02000000:00000001:0.0:1784949383.605358:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949383.605358:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4306322948 00000100:00000001:0.0:1784949383.605366:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00100000:2.0:1784949383.605375:0:337113:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:0.0:1784949383.605376:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949383.605379:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306322987 02000000:00000001:0.0:1784949383.605388:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949383.605392:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4306322948 00000100:00100000:0.0:1784949383.605395:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693568 00000020:00000001:0.0:1784949383.605401:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1784949383.605402:0:337113:0:(events.c:373:request_in_callback()) Process leaving 00000020:00000040:0.0:1784949383.605406:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:0.0:1784949383.605414:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1784949383.605416:0:337122:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306322987 00000800:00000001:2.0:1784949383.605418:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000040:0.0:1784949383.605419:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:0.0:1784949383.605429:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000800:00000001:2.0:1784949383.605430:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949383.605432:0:337122:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4306322948 00000020:00000001:0.0:1784949383.605440:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949383.605448:0:337122:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949383.605449:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949383.605459:0:337122:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949383.605460:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:0.0:1784949383.605469:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949383.605472:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949383.605475:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00000001:3.0:1784949383.605484:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949383.605489:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001c9428b0. 00000020:00000010:0.0:1784949383.605499:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000085586104. 00000100:00000001:3.0:1784949383.605499:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949383.605508:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:0.0:1784949383.605511:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000075e36e9f. 00000100:00000001:3.0:1784949383.605519:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949383.605525:0:342762:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:3.0:1784949383.605526:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1784949383.605528:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00100000:1.0:1784949383.605535:0:342762:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693696 00000100:00000001:3.0:1784949383.605537:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949383.605538:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 02000000:00000001:1.0:1784949383.605542:0:342762:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949383.605544:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:3.0:1784949383.605550:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949383.605551:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.605551:0:342762:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949383.605558:0:342762:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949383.605567:0:342762:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949383.605568:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949383.605571:0:342762:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693696 00000020:00000001:1.0:1784949383.605577:0:342762:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1784949383.605580:0:342762:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:1.0:1784949383.605583:0:342762:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1784949383.605587:0:342762:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:1.0:1784949383.605592:0:342762:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:0.0:1784949383.605597:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1784949383.605597:0:342762:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:1.0:1784949383.605611:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949383.605618:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000556e55b. 00000100:00000001:0.0:1784949383.605621:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:1.0:1784949383.605624:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949383.605626:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000020:00000010:1.0:1784949383.605634:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000556e55b. 00000100:00100000:0.0:1784949383.605639:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 841 00000100:00000040:0.0:1784949383.605647:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000020:00000010:1.0:1784949383.605649:0:342762:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000032fe57fd. 00000100:00000001:0.0:1784949383.605650:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782160976 : -115538927390640 : ffff96eaff467050) 00000020:00000010:1.0:1784949383.605661:0:342762:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000c17394e5. 00000100:00000040:0.0:1784949383.605663:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000000ae6b462 x1871651289693568/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:289/0 lens 224/0 e 0 to 0 dl 1784949394 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1784949383.605670:0:342762:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000000b70bed3. 00000100:00000001:0.0:1784949383.605685:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:1.0:1784949383.605686:0:342762:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000020:00000010:0.0:1784949383.605691:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000abb2992. 00000100:00000001:1.0:1784949383.605693:0:342762:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949383.605698:0:342762:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949383.605700:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949383.605704:0:342762:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949383.605706:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000abb2992. 00000100:00100000:0.0:1784949383.605717:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000000ae6b462 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289693568:12345-0@lo:400: 00000100:00000001:1.0:1784949383.605718:0:342762:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:1.0:1784949383.605724:0:342762:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000200:0.0:1784949383.605727:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693568 00000100:00000001:1.0:1784949383.605729:0:342762:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949383.605734:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949383.605740:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949383.605743:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949383.605746:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949383.605748:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:0.0:1784949383.605752:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:1.0:1784949383.605754:0:342762:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949383.605755:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949383.605766:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949383.605769:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949383.605796:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.605799:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000020:00000001:0.0:1784949383.605801:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949383.605808:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:0.0:1784949383.605810:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949383.605817:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00100000:1.0:1784949383.605824:0:342762:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 817 00000100:00000001:2.0:1784949383.605825:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 02000000:00000010:0.0:1784949383.605826:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000006926284c. 00000100:00000001:2.0:1784949383.605834:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 02000000:00000001:0.0:1784949383.605835:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949383.605836:0:342762:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949383.605841:0:342764:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949383.605845:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.605847:0:342762:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529469572864 : -115544239978752 : ffff96e9c29ec700) 00000100:00000001:2.0:1784949383.605850:0:342764:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949383.605858:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949383.605865:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003901a72b x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:289/0 lens 224/0 e 0 to 0 dl 1784949394 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00000040:0.0:1784949383.605866:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693568 00010000:00000001:0.0:1784949383.605875:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949383.605887:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000000ae6b462 x1871651289693568/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:289/0 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949383.605888:0:342762:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:1.0:1784949383.605897:0:342762:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000000556e55b. 00010000:00000001:0.0:1784949383.605939:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949383.605943:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949383.605950:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=105 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949383.605956:0:342762:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949383.605962:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:1.0:1784949383.605963:0:342762:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000000556e55b. 00000100:00000040:0.0:1784949383.605975:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00100000:1.0:1784949383.605980:0:342762:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003901a72b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289693696:12345-192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949383.605985:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00000200:1.0:1784949383.605991:0:342762:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693696 02000000:00000001:0.0:1784949383.605996:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:1.0:1784949383.605998:0:342762:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 02000000:00000001:0.0:1784949383.606004:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949383.606005:0:342762:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:1.0:1784949383.606010:0:342762:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949383.606016:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1784949383.606016:0:342762:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:1.0:1784949383.606024:0:342762:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000040:0.0:1784949383.606030:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000020:00000001:1.0:1784949383.606032:0:342762:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:1.0:1784949383.606038:0:342762:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1784949383.606040:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000020:00000001:1.0:1784949383.606042:0:342762:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:1.0:1784949383.606046:0:342762:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:1.0:1784949383.606051:0:342762:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949383.606054:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289693568, offset 224 00000020:00000001:1.0:1784949383.606055:0:342762:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949383.606062:0:342762:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:1.0:1784949383.606065:0:342762:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000400:00000200:0.0:1784949383.606067:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000010:1.0:1784949383.606073:0:342762:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000014f50db5. 02000000:00000001:1.0:1784949383.606079:0:342762:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949383.606086:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:1.0:1784949383.606088:0:342762:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949383.606096:0:342762:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1784949383.606099:0:342762:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693696 00010000:00000001:1.0:1784949383.606103:0:342762:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000400:00000200:0.0:1784949383.606108:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643fd80 00010000:00000200:1.0:1784949383.606114:0:342762:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003901a72b x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:289/0 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949383.606124:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9855 [1] + 224 00000400:00000200:0.0:1784949383.606139:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1784949383.606142:0:342762:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000400:00000200:0.0:1784949383.606148:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00010000:00000001:1.0:1784949383.606149:0:342762:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949383.606154:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000100:00001000:1.0:1784949383.606155:0:342762:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=100 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949383.606164:0:342762:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1784949383.606165:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949383.606173:0:342762:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949383.606181:0:342762:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:1.0:1784949383.606189:0:342762:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:1.0:1784949383.606193:0:342762:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.606202:0:342762:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949383.606214:0:342762:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:1.0:1784949383.606223:0:342762:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000100:00000040:0.0:1784949383.606224:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949383.606236:0:342762:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289693696, offset 224 00000400:00000200:1.0:1784949383.606250:0:342762:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:0.0:1784949383.606261:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1784949383.606267:0:342762:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:1.0:1784949383.606275:0:342762:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3954:3954:247:4294967295] 192.168.202.137@tcp LPNI seq info [2533:2533:8:4294967295] 00000400:00000200:1.0:1784949383.606290:0:342762:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:1.0:1784949383.606300:0:342762:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949383.606302:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000800:00000010:1.0:1784949383.606305:0:342762:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000005678171. 00000400:00000010:0.0:1784949383.606308:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:3.0:1784949383.606313:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949383.606318:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949383.606319:0:337122:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:1.0:1784949383.606327:0:342762:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:0.0:1784949383.606328:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000001:3.0:1784949383.606332:0:337122:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784949383.606336:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784949383.606338:0:337122:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000200:1.0:1784949383.606346:0:342762:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000400:00000010:3.0:1784949383.606347:0:337122:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001004a12d. 00000100:00000040:0.0:1784949383.606351:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000800:00000200:1.0:1784949383.606359:0:342762:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000005678171 type 1, nob 320 niov 1 nkiov 1 00010000:00000001:0.0:1784949383.606360:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000400:00000200:3.0:1784949383.606362:0:337122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bb55de7f 00000020:00000001:0.0:1784949383.606366:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949383.606372:0:337122:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bb55de7f. 00000020:00000001:0.0:1784949383.606373:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:3.0:1784949383.606382:0:337122:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1784949383.606384:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000ae6b462 x1871651289693568/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:289/0 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949383.606390:0:342762:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1784949383.606394:0:337122:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949383.606407:0:342762:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:1.0:1784949383.606419:0:342762:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:1.0:1784949383.606425:0:342762:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949383.606435:0:342762:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00100000:0.0:1784949383.606437:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000000ae6b462 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337122:x1871651289693568:12345-0@lo:400: Request processed in 745us (1380us total) trans 0 rc 0/0 00000100:00000040:1.0:1784949383.606448:0:342762:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003901a72b x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:289/0 lens 224/224 e 0 to 0 dl 1784949394 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1784949383.606457:0:337122:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949383.606470:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 841 00000100:00000040:0.0:1784949383.606483:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:0.0:1784949383.606492:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949383.606499:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949383.606509:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000085586104. 00000100:00000001:3.0:1784949383.606520:0:337122:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1784949383.606522:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000075e36e9f. 00000100:00000001:3.0:1784949383.606528:0:337122:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1784949383.606530:0:342762:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003901a72b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289693696:12345-192.168.202.137@tcp:400: Request processed in 633us (1164us total) trans 0 rc 0/0 00000020:00000010:0.0:1784949383.606533:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001c9428b0. 00000100:00000001:3.0:1784949383.606535:0:337122:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:3.0:1784949383.606541:0:337122:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949383.606548:0:337122:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000040:0.0:1784949383.606549:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:3.0:1784949383.606555:0:337122:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949383.606557:0:342762:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 817 02000000:00000001:0.0:1784949383.606560:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000001:3.0:1784949383.606563:0:337122:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949383.606568:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 000000006926284c. 00000100:00000040:1.0:1784949383.606569:0:342762:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00001000:3.0:1784949383.606571:0:337122:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=105 v=5 (5 5 5 5) 00000100:00000001:1.0:1784949383.606580:0:342762:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 02000000:00000001:0.0:1784949383.606581:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00001000:3.0:1784949383.606582:0:337122:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=105 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949383.606589:0:342762:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:0.0:1784949383.606590:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949383.606592:0:337122:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1784949383.606600:0:337122:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949383.606601:0:342762:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000c17394e5. 00000100:00000200:0.0:1784949383.606604:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00010000:00000001:3.0:1784949383.606605:0:337122:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000010:1.0:1784949383.606613:0:342762:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000000b70bed3. 00010000:00000001:3.0:1784949383.606614:0:337122:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1784949383.606617:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000055895529. 00000800:00000200:2.0:1784949383.606626:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949383.606627:0:337122:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1784949383.606630:0:342762:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000032fe57fd. 00000400:00000010:0.0:1784949383.606631:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006799df90. 00000800:00000010:2.0:1784949383.606638:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000005678171. 00000100:00000040:3.0:1784949383.606644:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1784949383.606647:0:342762:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000400:00000200:2.0:1784949383.606651:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1784949383.606659:0:342762:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949383.606675:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949383.606682:0:337122:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1784949383.606688:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a2727e1c 00000100:00000001:3.0:1784949383.606689:0:337122:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000001:0.0:1784949383.606691:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949383.606694:0:337122:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949383.606700:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a2727e1c. 00000100:00000040:3.0:1784949383.606705:0:337122:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1784949383.606709:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949383.606714:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:0.0:1784949383.606720:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949383.606724:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:0.0:1784949383.606729:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1784949383.606743:0:337122:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@0000000000b40f51 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337122:1871651289693568:0@lo:400: 00000400:00000200:0.0:1784949383.606756:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000001:3.0:1784949383.606759:0:337122:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949383.606764:0:337122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1784949383.606769:0:337122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@0000000000b40f51 x1871651289693568/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949383.606800:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643fe00 00000400:00000200:0.0:1784949383.606819:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9859 [1] + 224 00000100:00000001:3.0:1784949383.606821:0:337122:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000001:0.0:1784949383.606827:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949383.606830:0:337122:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1784949383.606837:0:337122:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000035b54a67. 00000800:00000001:0.0:1784949383.606841:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949383.606847:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784949383.606852:0:337122:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:0.0:1784949383.606853:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000020:00000001:3.0:1784949383.606862:0:337122:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:0.0:1784949383.606864:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:3.0:1784949383.606871:0:337122:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000100:00000200:0.0:1784949383.606876:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949383.606884:0:337122:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1784949383.606892:0:337122:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000003cb28a6d. 00000100:00000040:0.0:1784949383.606898:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949383.606945:0:337122:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1784949383.606951:0:337122:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1784949383.606954:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000010:3.0:1784949383.606955:0:337122:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000000b40f51. 00000100:00000001:3.0:1784949383.606968:0:337122:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000800:00000001:0.0:1784949383.606972:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949383.606972:0:337122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949383.606980:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949383.606985:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949383.606993:0:337122:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949383.606998:0:337122:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949383.607004:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949383.607007:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949383.607008:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.607014:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949383.607022:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949383.607024:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949383.607028:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000400:00000200:1.0:1784949383.607035:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784949383.607038:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000100:00000001:1.0:1784949383.607041:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949383.607047:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949383.607059:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949383.607069:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949383.607071:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949383.607073:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949383.607076:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949383.607077:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949383.607079:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949383.607082:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949383.607088:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=100 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949383.607092:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=100 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949383.607095:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949383.607096:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949383.607099:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949383.607103:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949383.607112:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.607115:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949383.607117:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949383.607119:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949383.607121:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949383.607124:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949383.607132:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949383.607135:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949383.607137:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949383.607140:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949383.607148:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000c646352a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289693696:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949383.607153:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949383.607155:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949383.607158:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000c646352a x1871651289693696/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949383.607173:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949383.607175:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949383.607178:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000006841197d. 02000000:00000001:1.0:1784949383.607182:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949383.607184:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949383.607185:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949383.607188:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949383.607190:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000284ae7c9. 02000000:00000001:1.0:1784949383.607193:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949383.607195:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949383.607197:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c646352a. 00000100:00000001:1.0:1784949383.607201:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949383.607203:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949383.607205:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949383.607210:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949383.607212:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949383.607214:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949383.607217:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949383.648020:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306322987, at 4306322988 00000100:00000040:0.0:1784949383.648041:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306322987, at 4306322988 00000100:00000010:2.0:1784949383.648042:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000040:3.0:1784949383.648043:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306322987, at 4306322988 00000100:00000040:1.0:1784949383.648046:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306322987, at 4306322988 00000100:00000010:0.0:1784949383.648066:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f595b698. 00000100:00000010:3.0:1784949383.648069:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000010:1.0:1784949383.648081:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000300e92c9. 00000800:00000100:3.0:1784949383.796079:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000800:00000010:3.0:1784949383.796086:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000ce8f840d. 00000400:00000200:3.0:1784949383.796097:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949383.796104:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949383.796112:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11656 health :900 00000400:00000200:3.0:1784949383.796118:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949383.796122:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:3.0:1784949383.796127:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:1.0:1784949383.796200:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949383.796212:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000001:1.0:1784949383.796220:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949383.796223:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949383.796227:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:1.0:1784949383.796229:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000400:00000200:1.0:1784949383.796235:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949383.796240:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000010:1.0:1784949383.796245:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000461737cb. 00000400:00000200:1.0:1784949383.796252:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:1.0:1784949383.796255:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000400:00000200:1.0:1784949383.796259:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00010000:00000001:2.0:1784949384.436038:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949384.436046:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949384.436053:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949384.436054:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949384.436152:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949384.436158:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949384.436161:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949384.628024:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949384.628040:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949384.628042:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949384.628047:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949384.628051:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949384.628055:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949384.628058:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949384.820079:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000800:00000100:3.0:1784949384.820082:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.11@tcp 00000800:00000010:3.0:1784949384.820091:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000676e0274. 00000400:00000010:2.0:1784949384.820094:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000ccf790c9. 00000400:00000200:3.0:1784949384.820103:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000010:2.0:1784949384.820108:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000200:3.0:1784949384.820118:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:2.0:1784949384.820122:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d4649544. 00000400:00000200:3.0:1784949384.820131:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11657 health :800 00000400:00000010:2.0:1784949384.820140:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000efd891ec. 00000400:00000200:3.0:1784949384.820142:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949384.820153:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.11@tcp: 1 00000400:00000200:2.0:1784949384.820161:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949384.820167:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000200:2.0:1784949384.820208:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949384.820238:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949384.820246:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3955:3955:248:4294967295] 192.168.202.137@tcp LPNI seq info [2534:2534:8:4294967295] 00000400:00000200:2.0:1784949384.820258:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:2.0:1784949384.820272:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:2.0:1784949384.820282:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949384.820291:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f15a9a04. 00000400:00000200:1.0:1784949384.820293:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000200:2.0:1784949384.820307:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000200:1.0:1784949384.820311:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x861 00000400:00000001:1.0:1784949384.820326:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000200:2.0:1784949384.820331:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000001:1.0:1784949384.820334:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949384.820351:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d6365940 00000800:00000200:2.0:1784949384.820355:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f15a9a04 type 2, nob 96 niov 1 nkiov 0 00000400:00000010:1.0:1784949384.820359:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d6365940. 00000400:00000200:1.0:1784949384.820370:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949384.820379:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: -114 00000400:00000010:1.0:1784949384.820386:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ead2c724. 00000400:00000200:1.0:1784949384.820397:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.11@tcp:-110 00000400:00000200:1.0:1784949384.820403:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 rc -110 00000400:00000200:1.0:1784949384.820412:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.11@tcp 00000800:00000200:3.0:1784949384.820573:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949384.820581:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f15a9a04. 00000400:00000200:3.0:1784949384.820591:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:2.0:1784949384.820592:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949384.820601:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949384.820606:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949384.820610:0:337113:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:2.0:1784949384.820611:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949384.820618:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949384.820638:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949384.820649:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:2.0:1784949384.820664:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:2.0:1784949384.820675:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:2.0:1784949384.820686:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949384.820697:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3956:3956:249:4294967295] 192.168.202.137@tcp LPNI seq info [2535:2535:8:4294967295] 00000400:00000200:2.0:1784949384.820707:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:2.0:1784949384.820712:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949384.820717:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000009662d9a. 00000800:00000200:2.0:1784949384.820722:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949384.820744:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:2.0:1784949384.820748:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000009662d9a type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949384.820896:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949384.820951:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:2.0:1784949384.820954:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949384.820961:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1784949384.820962:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000009662d9a. 00000800:00000001:3.0:1784949384.820968:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949384.820973:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949384.820977:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949384.820986:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949384.820992:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949384.821008:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:3.0:1784949384.821045:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9871 00000800:00000001:3.0:1784949384.821055:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949384.821070:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949384.821075:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949384.821081:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:3.0:1784949384.821089:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d4649544 00000400:00000010:3.0:1784949384.821093:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d4649544. 00000400:00000200:3.0:1784949384.821097:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:3.0:1784949384.821100:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000ccf790c9. 00000400:00000010:3.0:1784949384.821104:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000800:00000001:3.0:1784949384.821114:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949384.821117:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949385.459985:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949385.459991:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949385.459996:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949385.459997:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949385.460038:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949385.460042:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949385.460044:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949385.651970:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949385.651984:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949385.651986:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949385.651991:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949385.651995:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949385.651999:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949385.652002:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949385.844023:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.15@tcp 00000800:00000010:3.0:1784949385.844028:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000008e487d72. 00000400:00000200:3.0:1784949385.844037:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949385.844042:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949385.844047:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11658 health :900 00000400:00000200:3.0:1784949385.844052:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949385.844056:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.15@tcp: 1 00000400:00000200:3.0:1784949385.844059:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000200:1.0:1784949385.844109:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949385.844118:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x861 00000400:00000001:1.0:1784949385.844135:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949385.844137:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949385.844141:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d53ff17e 00000400:00000010:1.0:1784949385.844144:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d53ff17e. 00000400:00000200:1.0:1784949385.844149:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949385.844153:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: -114 00000400:00000010:1.0:1784949385.844158:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000be26f34f. 00000400:00000200:1.0:1784949385.844163:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.15@tcp:-110 00000400:00000200:1.0:1784949385.844167:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 rc -110 00000400:00000200:1.0:1784949385.844171:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.15@tcp 00010000:00000001:2.0:1784949386.484046:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949386.484056:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949386.484062:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949386.484063:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949386.484148:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949386.484155:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949386.484159:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949386.676020:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949386.676037:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949386.676040:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949386.676048:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949386.676051:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949386.676068:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949386.676071:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949386.868033:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000800:00000010:3.0:1784949386.868039:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c2b2d4f6. 00000400:00000200:3.0:1784949386.868051:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949386.868058:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949386.868078:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:3.0:1784949386.868093:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949386.868099:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:3.0:1784949386.868104:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:1.0:1784949386.868143:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1784949386.868152:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000001:1.0:1784949386.868160:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949386.868162:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949386.868178:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007c185147 00000400:00000010:1.0:1784949386.868180:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000007c185147. 00000400:00000200:1.0:1784949386.868185:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949386.868188:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000010:1.0:1784949386.868192:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000002c1a3d41. 00000400:00000200:1.0:1784949386.868198:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:1.0:1784949386.868201:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:1.0:1784949386.868204:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000010:2.0:1784949386.868217:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000066bd4e30. 00000400:00000010:2.0:1784949386.868228:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000010:2.0:1784949386.868234:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000abebc97a. 00000400:00000010:2.0:1784949386.868240:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bfee0de1. 00000400:00000200:2.0:1784949386.868246:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949386.868266:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949386.868273:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949386.868276:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3957:3957:251:4294967295] 192.168.202.137@tcp LPNI seq info [2536:2536:8:4294967295] 00000400:00000200:2.0:1784949386.868282:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:2.0:1784949386.868288:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:2.0:1784949386.868292:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949386.868297:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000009662d9a. 00000800:00000200:2.0:1784949386.868310:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949386.868318:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784949386.868322:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000009662d9a type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949386.868339:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11660 health :800 00000400:00000010:2.0:1784949386.868346:0:337116:0:(lib-move.c:3814:lnet_print_recovery_list()) alloc '(buf)': 65 at 00000000882090e1. 00000400:02000040:2.0:1784949386.868349:0:337116:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.202.137@tcp 00000800:00000001:3.0:1784949386.868475:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1784949386.868482:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949386.868488:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949386.868491:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949386.868496:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000009662d9a. 00000800:00000001:3.0:1784949386.868496:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949386.868505:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949386.868510:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:0.0:1784949386.868516:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949386.868527:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1784949386.868528:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:0.0:1784949386.868535:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000400:00000200:3.0:1784949386.868542:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949386.868555:0:337114:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949386.868559:0:337114:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3958:3958:251:4294967295] 192.168.202.137@tcp LPNI seq info [2537:2537:8:4294967295] 00000400:00000200:3.0:1784949386.868578:0:337114:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949386.868581:0:337114:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949386.868584:0:337114:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949386.868588:0:337114:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949386.868592:0:337114:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949386.868595:0:337114:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949386.868695:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949386.868710:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949386.868714:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000400:00000200:3.0:1784949386.868718:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949386.868722:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:0.0:1784949386.868732:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949386.868737:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949386.868756:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949386.868758:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949386.868776:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949386.868783:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949386.868804:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9875 00000800:00000001:0.0:1784949386.868809:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949386.868842:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949386.868844:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949386.868856:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949386.868860:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000abebc97a 00000400:00000010:0.0:1784949386.868862:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000abebc97a. 00000400:00000200:0.0:1784949386.868864:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949386.868866:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000066bd4e30. 00000400:00000010:0.0:1784949386.868879:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000800:00000001:0.0:1784949386.868886:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949386.868889:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949386.870433:0:337116:0:(lib-move.c:3832:lnet_print_recovery_list()) kfreed 'buf': 65 at 00000000882090e1. 00010000:00000001:2.0:1784949387.508065:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949387.508074:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949387.508083:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949387.508084:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949387.508185:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949387.508193:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949387.508199:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949387.700102:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949387.700120:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949387.700123:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949387.700131:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949387.700134:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949387.700139:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949387.700143:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1784949387.892073:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.30@tcp 00000800:00000010:3.0:1784949387.892081:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000e39f0ad5. 00000400:00000200:3.0:1784949387.892093:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949387.892100:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.30@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949387.892108:0:337111:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11660 health :900 00000400:00000200:3.0:1784949387.892114:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949387.892119:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.30@tcp: 1 00000400:00000200:3.0:1784949387.892125:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.30@tcp: -114 00000800:00000100:3.0:1784949387.892155:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.1@tcp 00000800:00000010:3.0:1784949387.892158:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000031deb17e. 00000400:00000200:3.0:1784949387.892164:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949387.892170:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949387.892175:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949387.892179:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.1@tcp: 1 00000400:00000200:3.0:1784949387.892185:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000800:00000100:3.0:1784949387.892197:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000800:00000010:3.0:1784949387.892201:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000012f4d1ec. 00000400:00000200:1.0:1784949387.892204:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949387.892206:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949387.892219:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949387.892223:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949387.892228:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.30@tcp(00000000229221cc) state 0x840 00000400:00000200:3.0:1784949387.892233:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:3.0:1784949387.892243:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000001:1.0:1784949387.892250:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.892257:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.892262:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000492bb2ef 00000400:00000010:1.0:1784949387.892268:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000492bb2ef. 00000400:00000200:1.0:1784949387.892278:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.892289:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.30@tcp: -114 00000400:00000010:1.0:1784949387.892299:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089e37819. 00000400:00000200:1.0:1784949387.892308:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.30@tcp:-110 00000400:00000200:1.0:1784949387.892313:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.30@tcp(00000000229221cc) state 0x2040 rc -110 00000400:00000200:1.0:1784949387.892320:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.30@tcp 00000400:00000200:1.0:1784949387.892359:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x861 00000400:00000001:1.0:1784949387.892363:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.892365:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.892367:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000aaaf29ff 00000400:00000010:1.0:1784949387.892369:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000aaaf29ff. 00000400:00000200:1.0:1784949387.892372:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.892376:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: -114 00000400:00000010:1.0:1784949387.892380:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ef8e57c8. 00000400:00000200:1.0:1784949387.892386:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.1@tcp:-110 00000400:00000200:1.0:1784949387.892390:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 rc -110 00000400:00000200:1.0:1784949387.892397:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.1@tcp 00000400:00000200:1.0:1784949387.892401:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000001:1.0:1784949387.892405:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.892408:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.892410:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000031068cb7 00000400:00000010:1.0:1784949387.892415:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000031068cb7. 00000400:00000200:3.0:1784949387.892419:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.30@tcp NID 111.222.142.30@tcp: -110. discovery complete 00000400:00000200:1.0:1784949387.892422:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.892437:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000200:3.0:1784949387.892441:0:343529:0:(peer.c:1550:LNetPrimaryNID()) NID 111.222.142.30@tcp primary NID 111.222.142.30@tcp rc -110 00000400:00000010:1.0:1784949387.892444:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000011003cc7. 00000400:00000200:1.0:1784949387.892453:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000100:00000010:3.0:1784949387.892469:0:343529:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000f5860ac6. 00000400:00000200:1.0:1784949387.892502:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000100:00000001:3.0:1784949387.892504:0:343529:0:(connection.c:145:ptlrpc_connection_get()) Process leaving 00000400:00000200:1.0:1784949387.892510:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000100:00000040:3.0:1784949387.892516:0:343529:0:(connection.c:149:ptlrpc_connection_get()) conn=00000000f5860ac6 refcount 1 to 111.222.142.30@tcp 00010000:00000010:3.0:1784949387.892525:0:343529:0:(ldlm_lib.c:87:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000077cd14ae. 00010000:00080000:3.0:1784949387.892595:0:343529:0:(ldlm_lib.c:125:import_set_conn()) imp 0000000036c7f24e@MGC192.168.202.137@tcp: add connection 111.222.142.30@tcp at tail 00010000:00000001:3.0:1784949387.892600:0:343529:0:(ldlm_lib.c:132:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949387.892604:0:343529:0:(obd_class.h:809:obd_add_conn()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1784949387.892619:0:343529:0:(lustre_peer.c:90:class_add_uuid()) kmalloced '(data)': 704 at 00000000a83fb011. 00000100:00000400:3.0:1784949387.892625:0:343529:0:(lustre_peer.c:124:class_add_uuid()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c: can't add NID 111.222.173.31@tcp: rc = -75 00000100:00000040:3.0:1784949387.892628:0:343529:0:(lustre_peer.c:130:class_add_uuid()) found uuid 1d45fc09-7805-4bc2-ab31-cf64613dd99c 111.222.173.31@tcp cnt=32 00000400:00000200:3.0:1784949387.892632:0:343529:0:(peer.c:1363:LNetAddPeer()) Add peer with 32 NIDs 00000400:00001000:3.0:1784949387.892636:0:343529:0:(api-ni.c:3331:LNetNIInit()) refs 1 00000400:00000200:3.0:1784949387.892640:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[0]: 0@lo 00000400:00000200:3.0:1784949387.892642:0:343529:0:(peer.c:1380:LNetAddPeer()) Skip NID 0@lo as local 00000400:00000200:3.0:1784949387.892644:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[1]: 192.168.202.137@tcp 00000400:00000200:3.0:1784949387.892652:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 192.168.202.137@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1784949387.892658:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.892662:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 192.168.202.137@tcp NID 192.168.202.137@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.892665:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 192.168.202.137@tcp 00000400:00000200:3.0:1784949387.892668:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[2]: 111.222.173.1@tcp 00000400:00000200:3.0:1784949387.892672:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.892691:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: 0 00000400:00000200:3.0:1784949387.892693:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.892696:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.892698:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.1@tcp NID 111.222.173.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.892701:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.1@tcp 00000400:00000200:3.0:1784949387.892704:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[3]: 111.222.142.1@tcp 00000400:00000200:3.0:1784949387.892708:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.1@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.892714:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.1@tcp: 0 00000400:00000200:3.0:1784949387.892723:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.892726:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.892731:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.1@tcp NID 111.222.142.1@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.892759:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.1@tcp 00000400:00000200:3.0:1784949387.892762:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[4]: 111.222.173.2@tcp 00000400:00000200:3.0:1784949387.892767:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.2@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.892771:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.2@tcp: 0 00000400:00000200:3.0:1784949387.892773:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.892775:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.892779:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.2@tcp NID 111.222.173.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.892783:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.2@tcp 00000400:00000200:1.0:1784949387.892787:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1784949387.892787:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[5]: 111.222.142.2@tcp 00000400:00000200:3.0:1784949387.892800:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.2@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.892805:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 00000400:00000200:3.0:1784949387.892814:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: 0 00000400:00000200:3.0:1784949387.892817:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.892822:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.892825:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.2@tcp NID 111.222.142.2@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.892828:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.2@tcp 00000400:00000010:1.0:1784949387.892829:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000032fe57fd. 00000400:00000200:3.0:1784949387.892831:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[6]: 111.222.173.3@tcp 00000400:00000200:3.0:1784949387.892840:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.3@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949387.892842:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000004238acf. 00000400:00000200:3.0:1784949387.892849:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: 0 00000400:00000200:3.0:1784949387.892853:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.892855:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.892858:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.3@tcp NID 111.222.173.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.892861:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.3@tcp 00000400:00000200:3.0:1784949387.892862:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[7]: 111.222.142.3@tcp 00000400:00000010:1.0:1784949387.892866:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001fcbecf4. 00000400:00000200:3.0:1784949387.892868:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.3@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.892884:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.3@tcp: 0 00000400:00000200:1.0:1784949387.892885:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949387.892888:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.892895:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.892898:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.3@tcp NID 111.222.142.3@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.892900:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.3@tcp 00000400:00000200:1.0:1784949387.892946:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.1@tcp local destination 00000400:00000200:3.0:1784949387.892950:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[8]: 111.222.173.4@tcp 00000400:00000200:1.0:1784949387.892958:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1784949387.892962:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.4@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.892971:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.892977:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.892980:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.1@tcp 00000400:00000200:1.0:1784949387.892985:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3959:3959:254:4294967295] 111.222.173.1@tcp LPNI seq info [56:56:8:4294967295] 00000400:00000200:1.0:1784949387.892993:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.1@tcp 00000400:00000200:1.0:1784949387.893001:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.1@tcp(111.222.173.1@tcp:111.222.173.1@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.893019:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.1@tcp 00000400:00000200:3.0:1784949387.893025:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.4@tcp: 0 00000800:00000010:1.0:1784949387.893027:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000005678171. 00000400:00000200:3.0:1784949387.893031:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893037:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949387.893045:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949387.893047:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.4@tcp NID 111.222.173.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893057:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.4@tcp 00000400:00000200:3.0:1784949387.893064:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[9]: 111.222.142.4@tcp 00000800:00000200:1.1:1784949387.893067:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000347f840b] -> 12345-111.222.173.1@tcp (2) 00000400:00000200:3.0:1784949387.893075:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.4@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.893082:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.1@tcp 00000400:00000200:3.0:1784949387.893089:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: 0 00000400:00000200:1.0:1784949387.893094:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x261 rc 0 00000400:00000200:3.0:1784949387.893097:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893108:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949387.893112:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x2061 00000400:00000200:3.0:1784949387.893120:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.4@tcp NID 111.222.142.4@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893126:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.4@tcp 00000400:00000200:3.0:1784949387.893130:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[10]: 111.222.173.5@tcp 00000400:00000010:1.0:1784949387.893131:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ef8e57c8. 00000400:00000010:1.0:1784949387.893141:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ecb574ad. 00000400:00000200:3.0:1784949387.893142:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.5@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949387.893153:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000da72cb2d. 00000400:00000200:3.0:1784949387.893158:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: 0 00000400:00000200:3.0:1784949387.893164:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893169:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949387.893170:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949387.893179:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.5@tcp NID 111.222.173.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893184:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.5@tcp 00000400:00000200:3.0:1784949387.893186:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[11]: 111.222.142.5@tcp 00000400:00000200:1.0:1784949387.893192:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.1@tcp local destination 00000400:00000200:3.0:1784949387.893194:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.5@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.893209:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.893216:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.893221:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.893226:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.1@tcp 00000400:00000200:1.0:1784949387.893231:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3960:3960:253:4294967295] 111.222.142.1@tcp LPNI seq info [54:54:8:4294967295] 00000400:00000200:1.0:1784949387.893241:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.1@tcp 00000400:00000200:1.0:1784949387.893252:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.1@tcp(111.222.142.1@tcp:111.222.142.1@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.893263:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.1@tcp 00000400:00000200:3.0:1784949387.893267:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000200:3.0:1784949387.893275:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893277:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000010:1.0:1784949387.893279:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f495c80e. 00000400:00000200:3.0:1784949387.893281:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.5@tcp NID 111.222.142.5@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893290:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.5@tcp 00000400:00000200:3.0:1784949387.893294:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[12]: 111.222.173.6@tcp 00000800:00000200:1.0:1784949387.893302:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949387.893304:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.6@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.893325:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.6@tcp: 0 00000800:00000200:1.1:1784949387.893325:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000003d54ad76] -> 12345-111.222.142.1@tcp (2) 00000400:00000200:3.0:1784949387.893332:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893338:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949387.893343:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.1@tcp 00000400:00000200:3.0:1784949387.893351:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.6@tcp NID 111.222.173.6@tcp: 0. pending discovery 00000400:00000200:1.0:1784949387.893356:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.1@tcp(0000000032c7e8f1) state 0x261 rc 0 00000400:00000200:3.0:1784949387.893360:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.6@tcp 00000400:00000200:3.0:1784949387.893369:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[13]: 111.222.142.6@tcp 00000400:00000200:1.0:1784949387.893370:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x2061 00000400:00000200:3.0:1784949387.893379:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.6@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949387.893380:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089e37819. 00000400:00000010:1.0:1784949387.893389:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000db7d3f50. 00000400:00000200:3.0:1784949387.893392:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: 0 00000400:00000200:3.0:1784949387.893397:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949387.893402:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000c3236cd0. 00000400:00000200:3.0:1784949387.893403:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.893415:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.6@tcp NID 111.222.142.6@tcp: 0. pending discovery 00000400:00000200:1.0:1784949387.893425:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.2@tcp 00000400:00000200:3.0:1784949387.893426:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.6@tcp 00000400:00000200:3.0:1784949387.893436:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[14]: 111.222.173.7@tcp 00000400:00000200:1.0:1784949387.893447:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.2@tcp local destination 00000400:00000200:3.0:1784949387.893448:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.7@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.893461:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.893466:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.893469:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.893473:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.2@tcp 00000400:00000200:1.0:1784949387.893476:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3961:3961:252:4294967295] 111.222.173.2@tcp LPNI seq info [53:53:8:4294967295] 00000400:00000200:1.0:1784949387.893481:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.2@tcp 00000400:00000200:1.0:1784949387.893487:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.2@tcp(111.222.173.2@tcp:111.222.173.2@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.893493:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.2@tcp 00000800:00000010:1.0:1784949387.893497:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000094ed938f. 00000400:00000200:3.0:1784949387.893500:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: 0 00000400:00000200:3.0:1784949387.893505:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893511:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949387.893512:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949387.893523:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.7@tcp NID 111.222.173.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893534:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.7@tcp 00000800:00000200:1.1:1784949387.893534:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000009c9de888] -> 12345-111.222.173.2@tcp (2) 00000400:00000200:3.0:1784949387.893545:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[15]: 111.222.142.7@tcp 00000400:00000200:3.0:1784949387.893551:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.7@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.893556:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.7@tcp: 0 00000400:00000200:3.0:1784949387.893558:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893559:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.893562:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.7@tcp NID 111.222.142.7@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893564:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.7@tcp 00000400:00000200:3.0:1784949387.893565:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[16]: 111.222.173.8@tcp 00000400:00000200:3.0:1784949387.893568:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.8@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.893572:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.8@tcp: 0 00000400:00000200:3.0:1784949387.893574:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893576:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.893580:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.8@tcp NID 111.222.173.8@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893582:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.8@tcp 00000400:00000200:3.0:1784949387.893584:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[17]: 111.222.142.8@tcp 00000400:00000200:3.0:1784949387.893587:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.8@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.893590:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: 0 00000400:00000200:3.0:1784949387.893593:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893596:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949387.893601:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.2@tcp 00000400:00000200:3.0:1784949387.893609:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.8@tcp NID 111.222.142.8@tcp: 0. pending discovery 00000400:00000200:1.0:1784949387.893616:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.2@tcp(000000004826ae4a) state 0x261 rc 0 00000400:00000200:3.0:1784949387.893619:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.8@tcp 00000400:00000200:3.0:1784949387.893627:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[18]: 111.222.173.9@tcp 00000400:00000200:1.0:1784949387.893632:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 00000400:00000200:3.0:1784949387.893640:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.9@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949387.893644:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ead2c724. 00000400:00000200:3.0:1784949387.893651:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: 0 00000400:00000010:1.0:1784949387.893655:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000200:3.0:1784949387.893657:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949387.893666:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000735cb99e. 00000400:00000200:3.0:1784949387.893667:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.893678:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.9@tcp NID 111.222.173.9@tcp: 0. pending discovery 00000400:00000200:1.0:1784949387.893688:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949387.893689:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.9@tcp 00000400:00000200:3.0:1784949387.893697:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[19]: 111.222.142.9@tcp 00000400:00000200:1.0:1784949387.893704:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.2@tcp local destination 00000400:00000200:3.0:1784949387.893706:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.9@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.893724:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.893729:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.893732:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.893761:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.2@tcp 00000400:00000200:1.0:1784949387.893765:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3962:3962:251:4294967295] 111.222.142.2@tcp LPNI seq info [53:53:8:4294967295] 00000400:00000200:1.0:1784949387.893772:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.2@tcp 00000400:00000200:1.0:1784949387.893779:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.2@tcp(111.222.142.2@tcp:111.222.142.2@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.893785:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.2@tcp 00000800:00000010:1.0:1784949387.893791:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000400:00000200:3.0:1784949387.893795:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.9@tcp: 0 00000400:00000200:3.0:1784949387.893800:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893808:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949387.893809:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000800:00000200:1.0:1784949387.893818:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784949387.893824:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.9@tcp NID 111.222.142.9@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893831:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.9@tcp 00000800:00000200:1.1:1784949387.893838:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000467b8b46] -> 12345-111.222.142.2@tcp (2) 00000400:00000200:3.0:1784949387.893840:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[20]: 111.222.173.10@tcp 00000800:00000200:1.1:1784949387.893845:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.2 (cnted 0, interval 60, 14 secs later) 00000400:00000200:3.0:1784949387.893851:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.10@tcp NID flags 0x1: -17 00000800:00000100:1.0:1784949387.893865:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.2@tcp 00000400:00000200:3.0:1784949387.893866:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.10@tcp: 0 00000800:00000010:1.0:1784949387.893873:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000200:3.0:1784949387.893875:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.893885:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949387.893886:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1784949387.893896:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.10@tcp NID 111.222.173.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893930:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.10@tcp 00000400:00000200:3.0:1784949387.893935:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[21]: 111.222.142.10@tcp 00000400:00000200:1.0:1784949387.893943:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.2@tcp: GET: REMOTE_ERROR 00000400:00000200:3.0:1784949387.893944:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.10@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.893959:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949387.893965:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: 0 00000400:00000200:3.0:1784949387.893970:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949387.893972:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.2@tcp: 1 00000400:00000200:3.0:1784949387.893975:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.893983:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.10@tcp NID 111.222.142.10@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.893986:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.10@tcp 00000400:00000200:3.0:1784949387.893988:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[22]: 111.222.173.11@tcp 00000400:00000200:3.0:1784949387.893992:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.11@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.893997:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000200:1.0:1784949387.894023:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.2@tcp 00000400:00000200:3.0:1784949387.894030:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:1.0:1784949387.894031:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 rc 0 00000400:00000200:3.0:1784949387.894037:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.894046:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949387.894050:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 00000400:00000200:3.0:1784949387.894060:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.11@tcp NID 111.222.173.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894065:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.11@tcp 00000400:00000010:1.0:1784949387.894069:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000461737cb. 00000400:00000200:3.0:1784949387.894070:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[23]: 111.222.142.11@tcp 00000400:00000010:1.0:1784949387.894081:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005c5c5c3. 00000400:00000200:3.0:1784949387.894082:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.11@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.894096:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.11@tcp: 0 00000400:00000200:3.0:1784949387.894098:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949387.894102:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000005547c433. 00000400:00000200:1.0:1784949387.894111:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.3@tcp 00000400:00000200:3.0:1784949387.894210:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.894218:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.11@tcp NID 111.222.142.11@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894222:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.11@tcp 00000400:00000200:3.0:1784949387.894224:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[24]: 111.222.173.12@tcp 00000400:00000200:1.0:1784949387.894238:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.3@tcp local destination 00000400:00000200:3.0:1784949387.894238:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.12@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.894254:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.894262:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.894268:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.894276:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.3@tcp 00000400:00000200:1.0:1784949387.894284:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3963:3963:251:4294967295] 111.222.173.3@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:1.0:1784949387.894296:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.3@tcp 00000400:00000200:1.0:1784949387.894310:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.3@tcp(111.222.173.3@tcp:111.222.173.3@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.894321:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.3@tcp 00000800:00000010:1.0:1784949387.894330:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000000475ac2. 00000400:00000200:3.0:1784949387.894331:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.12@tcp: 0 00000400:00000200:3.0:1784949387.894339:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.894344:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.894354:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.12@tcp NID 111.222.173.12@tcp: 0. pending discovery 00000800:00000200:1.0:1784949387.894356:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949387.894358:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.12@tcp 00000400:00000200:3.0:1784949387.894367:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[25]: 111.222.142.12@tcp 00000800:00000200:1.1:1784949387.894374:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002649f27f] -> 12345-111.222.173.3@tcp (2) 00000400:00000200:3.0:1784949387.894376:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.12@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.894392:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.3@tcp 00000400:00000200:3.0:1784949387.894393:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: 0 00000400:00000200:3.0:1784949387.894400:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:1.0:1784949387.894405:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x261 rc 0 00000400:00000200:3.0:1784949387.894407:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.894417:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.12@tcp NID 111.222.142.12@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894421:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.12@tcp 00000400:00000200:3.0:1784949387.894424:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[26]: 111.222.173.13@tcp 00000400:00000200:1.0:1784949387.894429:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x2061 00000400:00000200:3.0:1784949387.894437:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.13@tcp NID flags 0x1: -17 00000400:00000010:1.0:1784949387.894440:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000be26f34f. 00000400:00000200:3.0:1784949387.894448:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: 0 00000400:00000010:1.0:1784949387.894451:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003f05b564. 00000400:00000200:3.0:1784949387.894452:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.894459:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.894465:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.13@tcp NID 111.222.173.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894467:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.13@tcp 00000400:00000010:1.0:1784949387.894469:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009e875900. 00000400:00000200:3.0:1784949387.894471:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[27]: 111.222.142.13@tcp 00000400:00000200:3.0:1784949387.894487:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.13@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.894490:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.3@tcp 00000400:00000200:3.0:1784949387.894497:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.13@tcp: 0 00000400:00000200:3.0:1784949387.894500:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.894505:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.894508:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.13@tcp NID 111.222.142.13@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894510:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.13@tcp 00000400:00000200:3.0:1784949387.894514:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[28]: 111.222.173.14@tcp 00000400:00000200:1.0:1784949387.894520:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.3@tcp local destination 00000400:00000200:3.0:1784949387.894527:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.14@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.894532:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.894539:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.894542:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.894545:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.3@tcp 00000400:00000200:1.0:1784949387.894549:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3964:3964:250:4294967295] 111.222.142.3@tcp LPNI seq info [50:50:8:4294967295] 00000400:00000200:1.0:1784949387.894554:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.3@tcp 00000400:00000200:1.0:1784949387.894561:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.3@tcp(111.222.142.3@tcp:111.222.142.3@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.894568:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.3@tcp 00000800:00000010:1.0:1784949387.894571:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000ca1124f5. 00000400:00000200:3.0:1784949387.894573:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.14@tcp: 0 00000400:00000200:3.0:1784949387.894580:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.894584:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000800:00000200:1.0:1784949387.894586:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949387.894596:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.14@tcp NID 111.222.173.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894604:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.14@tcp 00000800:00000200:1.1:1784949387.894605:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000401e6ce4] -> 12345-111.222.142.3@tcp (2) 00000400:00000200:3.0:1784949387.894610:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[29]: 111.222.142.14@tcp 00000400:00000200:3.0:1784949387.894615:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.14@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.894619:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: 0 00000400:00000200:3.0:1784949387.894620:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.894622:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.894625:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.14@tcp NID 111.222.142.14@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894634:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.14@tcp 00000400:00000200:3.0:1784949387.894637:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[30]: 111.222.173.15@tcp 00000400:00000200:1.0:1784949387.894642:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.3@tcp 00000400:00000200:3.0:1784949387.894649:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.173.15@tcp NID flags 0x1: -17 00000400:00000200:1.0:1784949387.894655:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.3@tcp(000000005f52e2e4) state 0x261 rc 0 00000400:00000200:3.0:1784949387.894663:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: 0 00000400:00000200:3.0:1784949387.894667:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1784949387.894672:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1784949387.894677:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x2061 00000400:00000200:3.0:1784949387.894681:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.173.15@tcp NID 111.222.173.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894688:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.173.15@tcp 00000400:00000010:1.0:1784949387.894691:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006841197d. 00000400:00000200:3.0:1784949387.894692:0:343529:0:(peer.c:1377:LNetAddPeer()) Adding NID[31]: 111.222.142.15@tcp 00000400:00000010:1.0:1784949387.894704:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000041a6162e. 00000400:00000200:3.0:1784949387.894705:0:343529:0:(peer.c:1822:lnet_peer_add()) peer 111.222.142.15@tcp NID flags 0x1: -17 00000400:00000200:3.0:1784949387.894724:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.15@tcp: 0 00000400:00000200:3.0:1784949387.894726:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949387.894730:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000038aa1d2e. 00000400:00000200:3.0:1784949387.894731:0:343529:0:(peer.c:2890:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1784949387.894763:0:343529:0:(peer.c:2899:lnet_discover_peer_locked()) peer 111.222.142.15@tcp NID 111.222.142.15@tcp: 0. pending discovery 00000400:00000200:3.0:1784949387.894768:0:343529:0:(peer.c:1431:LNetAddPeer()) A peer exists for NID 111.222.142.15@tcp 00000400:00000200:1.0:1784949387.894775:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.4@tcp 00000100:00000010:3.0:1784949387.894778:0:343529:0:(lustre_peer.c:132:class_add_uuid()) kfreed 'data': 704 at 00000000a83fb011. 00000020:00000001:3.0:1784949387.894791:0:343529:0:(obd_class.h:797:obd_add_conn()) Process entered 00000400:00000200:1.0:1784949387.894792:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.4@tcp local destination 00010000:00000001:3.0:1784949387.894799:0:343529:0:(ldlm_lib.c:52:import_set_conn()) Process entered 00000400:00000200:1.0:1784949387.894802:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1784949387.894807:0:343529:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:1.0:1784949387.894809:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.894815:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.894818:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.4@tcp 00000400:00000200:1.0:1784949387.894821:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3965:3965:249:4294967295] 111.222.173.4@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:1.0:1784949387.894826:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.4@tcp 00000400:00000200:1.0:1784949387.894832:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.4@tcp(111.222.173.4@tcp:111.222.173.4@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.894838:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.4@tcp 00000400:00000010:3.0:1784949387.894843:0:343529:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at 00000000bf393d23. 00000800:00000010:1.0:1784949387.894845:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 0000000076916763. 00000400:00000200:3.0:1784949387.894860:0:343529:0:(peer.c:266:lnet_peer_alloc()) 00000000bf393d23 nid 111.222.173.31@tcp 00000800:00000200:1.0:1784949387.894861:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000010:3.0:1784949387.894871:0:343529:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000003d48e6d9. 00000800:00000200:1.1:1784949387.894872:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000005977ed21] -> 12345-111.222.173.4@tcp (2) 00000400:00000200:3.0:1784949387.894885:0:343529:0:(peer.c:203:lnet_peer_net_alloc()) 000000003d48e6d9 net tcp 00000400:00000010:3.0:1784949387.894891:0:343529:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at 000000009f047fe3. 00000400:00000200:3.0:1784949387.894897:0:343529:0:(peer.c:184:lnet_peer_ni_alloc()) 000000009f047fe3 nid 111.222.173.31@tcp 00000400:00000200:1.0:1784949387.894941:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.4@tcp 00000400:00000200:3.0:1784949387.894942:0:343529:0:(peer.c:1719:lnet_peer_attach_peer_ni()) peer 111.222.173.31@tcp NID 111.222.173.31@tcp flags 0x0 00000400:00000200:1.0:1784949387.894954:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.4@tcp(00000000db8f1157) state 0x261 rc 0 00000400:00000200:3.0:1784949387.894960:0:343529:0:(peer.c:2098:lnet_peer_ni_traffic_add()) peer 111.222.173.31@tcp: 0 00000400:00000200:1.0:1784949387.894972:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 00000400:00000200:3.0:1784949387.894974:0:343529:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.31@tcp: 0 00000400:00000200:3.0:1784949387.894982:0:343529:0:(peer.c:2850:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000010:1.0:1784949387.894994:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000307ddfde. 00000400:00000010:1.0:1784949387.895004:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e5294b24. 00000400:00000010:1.0:1784949387.895022:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000147b89d3. 00000400:00000200:1.0:1784949387.895031:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.4@tcp 00000400:00000200:1.0:1784949387.895044:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.4@tcp local destination 00000400:00000200:1.0:1784949387.895051:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895057:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895061:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895066:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.4@tcp 00000400:00000200:1.0:1784949387.895072:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3966:3966:248:4294967295] 111.222.142.4@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:1.0:1784949387.895079:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.4@tcp 00000400:00000200:1.0:1784949387.895095:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.4@tcp(111.222.142.4@tcp:111.222.142.4@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895103:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.4@tcp 00000800:00000010:1.0:1784949387.895109:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:1.0:1784949387.895114:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.0:1784949387.895117:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949387.895125:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000036849a47] -> 12345-111.222.142.4@tcp (2) 00000800:00000200:1.1:1784949387.895128:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.4 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949387.895133:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.4@tcp 00000800:00000010:1.0:1784949387.895137:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:1.0:1784949387.895143:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949387.895148:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.4@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949387.895151:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949387.895154:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.4@tcp: 1 00000400:00000200:1.0:1784949387.895158:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000200:1.0:1784949387.895165:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.4@tcp 00000400:00000200:1.0:1784949387.895167:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 rc 0 00000400:00000200:1.0:1784949387.895171:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 00000400:00000010:1.0:1784949387.895176:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000194fc0c0. 00000400:00000010:1.0:1784949387.895179:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000600895c9. 00000400:00000010:1.0:1784949387.895182:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000067f2d25d. 00000400:00000200:1.0:1784949387.895186:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.5@tcp 00000400:00000200:1.0:1784949387.895190:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.5@tcp local destination 00000400:00000200:1.0:1784949387.895194:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895198:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895202:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895207:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.5@tcp 00000400:00000200:1.0:1784949387.895212:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3967:3967:248:4294967295] 111.222.173.5@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:1.0:1784949387.895220:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.5@tcp 00000400:00000200:1.0:1784949387.895231:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.5@tcp(111.222.173.5@tcp:111.222.173.5@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895240:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.5@tcp 00000800:00000010:1.0:1784949387.895245:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000009e7fe55b. 00000800:00000200:1.0:1784949387.895253:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000800:00000200:1.1:1784949387.895260:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000c186a3d2] -> 12345-111.222.173.5@tcp (2) 00000400:00000200:1.0:1784949387.895265:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.5@tcp 00000400:00000200:1.0:1784949387.895269:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x261 rc 0 00000400:00000200:1.0:1784949387.895276:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x2061 00000400:00000010:1.0:1784949387.895283:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000006620b0a1. 00000400:00000010:1.0:1784949387.895288:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000588d3b65. 00000400:00000010:1.0:1784949387.895295:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000dceddabe. 00000400:00000200:1.0:1784949387.895300:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.6@tcp 00000400:00000200:1.0:1784949387.895309:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.6@tcp local destination 00000400:00000200:1.0:1784949387.895316:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895321:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895326:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895331:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.6@tcp 00000400:00000200:1.0:1784949387.895335:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3968:3968:247:4294967295] 111.222.173.6@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:1.0:1784949387.895344:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.6@tcp 00000400:00000200:1.0:1784949387.895351:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.6@tcp(111.222.173.6@tcp:111.222.173.6@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895357:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.6@tcp 00000800:00000010:1.0:1784949387.895373:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000007a3f8609. 00000800:00000200:1.0:1784949387.895380:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000800:00000200:1.1:1784949387.895385:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006eeaa7e0] -> 12345-111.222.173.6@tcp (2) 00000400:00000200:1.0:1784949387.895408:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.6@tcp 00000400:00000200:1.0:1784949387.895411:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.6@tcp(00000000f03bb5d5) state 0x261 rc 0 00000400:00000200:1.0:1784949387.895415:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 00000400:00000010:1.0:1784949387.895420:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000038ac69a2. 00000400:00000010:1.0:1784949387.895423:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d777f653. 00000400:00000010:1.0:1784949387.895427:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000756d087e. 00000400:00000200:1.0:1784949387.895431:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.6@tcp 00000400:00000200:1.0:1784949387.895450:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.6@tcp local destination 00000400:00000200:1.0:1784949387.895458:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895464:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895470:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895476:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.6@tcp 00000400:00000200:1.0:1784949387.895483:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3969:3969:246:4294967295] 111.222.142.6@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:1.0:1784949387.895492:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.6@tcp 00000400:00000200:1.0:1784949387.895509:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.6@tcp(111.222.142.6@tcp:111.222.142.6@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895518:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.6@tcp 00000800:00000010:1.0:1784949387.895525:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b9b32818. 00000800:00000200:1.0:1784949387.895531:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.0:1784949387.895534:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949387.895539:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000008bd899a0] -> 12345-111.222.142.6@tcp (2) 00000800:00000200:1.1:1784949387.895542:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.6 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949387.895548:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.6@tcp 00000800:00000010:1.0:1784949387.895552:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b9b32818. 00000400:00000200:1.0:1784949387.895557:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949387.895561:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.6@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949387.895565:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949387.895568:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.6@tcp: 1 00000400:00000200:1.0:1784949387.895572:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000200:1.0:1784949387.895578:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.6@tcp 00000400:00000200:1.0:1784949387.895581:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 rc 0 00000400:00000200:1.0:1784949387.895587:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 00000400:00000010:1.0:1784949387.895593:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b105ef1a. 00000400:00000010:1.0:1784949387.895597:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000066cfac65. 00000400:00000010:1.0:1784949387.895601:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000096383a5. 00000400:00000200:1.0:1784949387.895605:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.7@tcp 00000400:00000200:1.0:1784949387.895611:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.7@tcp local destination 00000400:00000200:1.0:1784949387.895614:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895617:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895620:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895624:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.7@tcp 00000400:00000200:1.0:1784949387.895628:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3970:3970:246:4294967295] 111.222.173.7@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:1.0:1784949387.895633:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.7@tcp 00000400:00000200:1.0:1784949387.895641:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.7@tcp(111.222.173.7@tcp:111.222.173.7@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895646:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.7@tcp 00000800:00000010:1.0:1784949387.895649:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b9b32818. 00000800:00000200:1.0:1784949387.895653:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000800:00000200:1.1:1784949387.895657:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000012b7d74b] -> 12345-111.222.173.7@tcp (2) 00000400:00000200:1.0:1784949387.895660:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.7@tcp 00000400:00000200:1.0:1784949387.895662:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x261 rc 0 00000400:00000200:1.0:1784949387.895666:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x2061 00000400:00000010:1.0:1784949387.895670:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000dc7a25c6. 00000400:00000010:1.0:1784949387.895672:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bc970d6e. 00000400:00000010:1.0:1784949387.895676:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000004d10c17. 00000400:00000200:1.0:1784949387.895679:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.7@tcp 00000400:00000200:1.0:1784949387.895682:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.7@tcp local destination 00000400:00000200:1.0:1784949387.895686:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895688:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895691:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895694:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.7@tcp 00000400:00000200:1.0:1784949387.895697:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3971:3971:245:4294967295] 111.222.142.7@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:1.0:1784949387.895701:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.7@tcp 00000400:00000200:1.0:1784949387.895706:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.7@tcp(111.222.142.7@tcp:111.222.142.7@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895711:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.7@tcp 00000800:00000010:1.0:1784949387.895713:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cee99234. 00000800:00000200:1.0:1784949387.895726:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000800:00000200:1.1:1784949387.895729:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000087133b69] -> 12345-111.222.142.7@tcp (2) 00000400:00000200:1.0:1784949387.895732:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.7@tcp 00000400:00000200:1.0:1784949387.895758:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.7@tcp(00000000dd28d373) state 0x261 rc 0 00000400:00000200:1.0:1784949387.895763:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x2061 00000400:00000010:1.0:1784949387.895767:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001879354a. 00000400:00000010:1.0:1784949387.895770:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000af21ddf0. 00000400:00000010:1.0:1784949387.895774:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003c8b2437. 00000400:00000200:1.0:1784949387.895777:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.8@tcp 00000400:00000200:1.0:1784949387.895782:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.8@tcp local destination 00000400:00000200:1.0:1784949387.895785:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895788:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895790:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895793:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.8@tcp 00000400:00000200:1.0:1784949387.895796:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3972:3972:244:4294967295] 111.222.173.8@tcp LPNI seq info [44:44:8:4294967295] 00000400:00000200:1.0:1784949387.895801:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.8@tcp 00000400:00000200:1.0:1784949387.895807:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.8@tcp(111.222.173.8@tcp:111.222.173.8@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895813:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.8@tcp 00000800:00000010:1.0:1784949387.895817:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a21d8ef3. 00000800:00000200:1.0:1784949387.895821:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000800:00000200:1.1:1784949387.895825:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000fe0db58c] -> 12345-111.222.173.8@tcp (2) 00000400:00000200:1.0:1784949387.895841:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.8@tcp 00000400:00000200:1.0:1784949387.895844:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.8@tcp(000000002d085db1) state 0x261 rc 0 00000400:00000200:1.0:1784949387.895849:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 00000400:00000010:1.0:1784949387.895858:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000053ab16eb. 00000400:00000010:1.0:1784949387.895862:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ed151fd1. 00000400:00000010:1.0:1784949387.895868:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000a7ce911a. 00000400:00000200:1.0:1784949387.895872:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.8@tcp 00000400:00000200:1.0:1784949387.895879:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.8@tcp local destination 00000400:00000200:1.0:1784949387.895884:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.895887:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.895891:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.895896:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.8@tcp 00000400:00000200:1.0:1784949387.895899:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3973:3973:243:4294967295] 111.222.142.8@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:1.0:1784949387.895947:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.8@tcp 00000400:00000200:1.0:1784949387.895957:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.8@tcp(111.222.142.8@tcp:111.222.142.8@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.895966:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.8@tcp 00000800:00000010:1.0:1784949387.895972:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000800:00000200:1.0:1784949387.895979:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.0:1784949387.895983:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949387.895989:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002262b69f] -> 12345-111.222.142.8@tcp (2) 00000800:00000200:1.1:1784949387.895992:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.8 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949387.895996:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.8@tcp 00000800:00000010:1.0:1784949387.895999:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000400:00000200:1.0:1784949387.896004:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949387.896020:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.8@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949387.896024:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949387.896030:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.8@tcp: 1 00000400:00000200:1.0:1784949387.896034:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000200:1.0:1784949387.896048:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.8@tcp 00000400:00000200:1.0:1784949387.896054:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 rc 0 00000400:00000200:1.0:1784949387.896063:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 00000400:00000010:1.0:1784949387.896069:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000fff5e130. 00000400:00000010:1.0:1784949387.896073:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e0ff87dd. 00000400:00000010:1.0:1784949387.896077:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003d75c981. 00000400:00000200:1.0:1784949387.896083:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.9@tcp 00000400:00000200:1.0:1784949387.896089:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.9@tcp local destination 00000400:00000200:1.0:1784949387.896093:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896097:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896099:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896102:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.9@tcp 00000400:00000200:1.0:1784949387.896105:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3974:3974:243:4294967295] 111.222.173.9@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:1.0:1784949387.896110:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.9@tcp 00000400:00000200:1.0:1784949387.896116:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.9@tcp(111.222.173.9@tcp:111.222.173.9@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896120:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.9@tcp 00000800:00000010:1.0:1784949387.896123:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000001ee120ca. 00000800:00000200:1.0:1784949387.896127:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000800:00000200:1.1:1784949387.896131:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000dea02e95] -> 12345-111.222.173.9@tcp (2) 00000400:00000200:1.0:1784949387.896135:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.9@tcp 00000400:00000200:1.0:1784949387.896137:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x261 rc 0 00000400:00000200:1.0:1784949387.896140:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x2061 00000400:00000010:1.0:1784949387.896145:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000b1cbf66f. 00000400:00000010:1.0:1784949387.896147:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007d821c01. 00000400:00000010:1.0:1784949387.896151:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000383d6bd1. 00000400:00000200:1.0:1784949387.896154:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.9@tcp 00000400:00000200:1.0:1784949387.896158:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.9@tcp local destination 00000400:00000200:1.0:1784949387.896161:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896164:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896166:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896169:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.9@tcp 00000400:00000200:1.0:1784949387.896172:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3975:3975:242:4294967295] 111.222.142.9@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:1.0:1784949387.896176:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.9@tcp 00000400:00000200:1.0:1784949387.896182:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.9@tcp(111.222.142.9@tcp:111.222.142.9@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896186:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.9@tcp 00000800:00000010:1.0:1784949387.896189:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c53910d9. 00000800:00000200:1.0:1784949387.896193:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000800:00000200:1.1:1784949387.896199:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006e067d3a] -> 12345-111.222.142.9@tcp (2) 00000400:00000200:1.0:1784949387.896206:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.9@tcp 00000400:00000200:1.0:1784949387.896209:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.9@tcp(0000000031640f0a) state 0x261 rc 0 00000400:00000200:1.0:1784949387.896215:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x2061 00000400:00000010:1.0:1784949387.896220:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000059e5b358. 00000400:00000010:1.0:1784949387.896225:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076145f95. 00000400:00000010:1.0:1784949387.896232:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000001f662712. 00000400:00000200:1.0:1784949387.896240:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.10@tcp 00000400:00000200:1.0:1784949387.896252:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.10@tcp local destination 00000400:00000200:1.0:1784949387.896257:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896261:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896265:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896270:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.10@tcp 00000400:00000200:1.0:1784949387.896275:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3976:3976:241:4294967295] 111.222.173.10@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:1.0:1784949387.896283:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.10@tcp 00000400:00000200:1.0:1784949387.896293:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.10@tcp(111.222.173.10@tcp:111.222.173.10@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896301:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.10@tcp 00000800:00000010:1.0:1784949387.896306:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000cf155122. 00000800:00000200:1.0:1784949387.896311:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000800:00000200:1.1:1784949387.896316:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006fc3cd6c] -> 12345-111.222.173.10@tcp (2) 00000400:00000200:1.0:1784949387.896335:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.10@tcp 00000400:00000200:1.0:1784949387.896338:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.10@tcp(000000009c28ea88) state 0x261 rc 0 00000400:00000200:1.0:1784949387.896342:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 00000400:00000010:1.0:1784949387.896347:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000f6f633a2. 00000400:00000010:1.0:1784949387.896352:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000400:00000010:1.0:1784949387.896358:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000bfef55e5. 00000400:00000200:1.0:1784949387.896365:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.10@tcp 00000400:00000200:1.0:1784949387.896371:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.10@tcp local destination 00000400:00000200:1.0:1784949387.896375:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896378:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896381:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896383:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.10@tcp 00000400:00000200:1.0:1784949387.896387:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3977:3977:240:4294967295] 111.222.142.10@tcp LPNI seq info [38:38:8:4294967295] 00000400:00000200:1.0:1784949387.896392:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.10@tcp 00000400:00000200:1.0:1784949387.896398:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.10@tcp(111.222.142.10@tcp:111.222.142.10@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896402:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.10@tcp 00000800:00000010:1.0:1784949387.896408:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000800:00000200:1.0:1784949387.896413:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.0:1784949387.896418:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949387.896423:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000740e81be] -> 12345-111.222.142.10@tcp (2) 00000800:00000200:1.1:1784949387.896426:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.10 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949387.896432:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.10@tcp 00000800:00000010:1.0:1784949387.896435:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000a934b7fa. 00000400:00000200:1.0:1784949387.896439:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949387.896443:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.10@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949387.896446:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949387.896449:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.10@tcp: 1 00000400:00000200:1.0:1784949387.896453:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000200:1.0:1784949387.896459:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.10@tcp 00000400:00000200:1.0:1784949387.896462:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 rc 0 00000400:00000200:1.0:1784949387.896467:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x861 00000400:00000001:1.0:1784949387.896469:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.896471:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.896473:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004f66310a 00000400:00000010:1.0:1784949387.896475:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004f66310a. 00000400:00000200:1.0:1784949387.896479:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.896482:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.2@tcp: -114 00000400:00000010:1.0:1784949387.896486:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ead2c724. 00000400:00000200:1.0:1784949387.896493:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.2@tcp:-5 00000400:00000200:1.0:1784949387.896496:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.2@tcp(000000008d199fbc) state 0x2061 rc -5 00000400:00000200:1.0:1784949387.896500:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.2@tcp 00000400:00000200:1.0:1784949387.896505:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x2061 00000400:00000010:1.0:1784949387.896510:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ead2c724. 00000400:00000010:1.0:1784949387.896513:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004f66310a. 00000400:00000010:1.0:1784949387.896516:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000054872d18. 00000400:00000200:1.0:1784949387.896520:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.11@tcp 00000400:00000200:1.0:1784949387.896524:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.11@tcp local destination 00000400:00000200:1.0:1784949387.896527:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896530:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896532:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896535:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.11@tcp 00000400:00000200:1.0:1784949387.896539:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3978:3978:240:4294967295] 111.222.142.11@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:1.0:1784949387.896544:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.11@tcp 00000400:00000200:1.0:1784949387.896550:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.11@tcp(111.222.142.11@tcp:111.222.142.11@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896555:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.11@tcp 00000800:00000010:1.0:1784949387.896558:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a934b7fa. 00000800:00000200:1.0:1784949387.896562:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000800:00000200:1.1:1784949387.896566:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000002df672a8] -> 12345-111.222.142.11@tcp (2) 00000400:00000200:1.0:1784949387.896569:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.11@tcp 00000400:00000200:1.0:1784949387.896571:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.11@tcp(000000004b1731aa) state 0x261 rc 0 00000400:00000200:1.0:1784949387.896574:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x2061 00000400:00000010:1.0:1784949387.896578:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000005ae2686. 00000400:00000010:1.0:1784949387.896580:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cdea04e8. 00000400:00000010:1.0:1784949387.896584:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000b03d8085. 00000400:00000200:1.0:1784949387.896587:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.12@tcp 00000400:00000200:1.0:1784949387.896591:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.12@tcp local destination 00000400:00000200:1.0:1784949387.896594:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896596:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896599:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896601:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.12@tcp 00000400:00000200:1.0:1784949387.896604:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3979:3979:239:4294967295] 111.222.173.12@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:1.0:1784949387.896608:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.12@tcp 00000400:00000200:1.0:1784949387.896614:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.12@tcp(111.222.173.12@tcp:111.222.173.12@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896618:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.12@tcp 00000800:00000010:1.0:1784949387.896622:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c158c1d0. 00000800:00000200:1.0:1784949387.896626:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000800:00000200:1.1:1784949387.896630:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001b171c73] -> 12345-111.222.173.12@tcp (2) 00000400:00000200:1.0:1784949387.896645:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.12@tcp 00000400:00000200:1.0:1784949387.896647:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.12@tcp(00000000b4d1439f) state 0x261 rc 0 00000400:00000200:1.0:1784949387.896650:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 00000400:00000010:1.0:1784949387.896654:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000000f0e4e38. 00000400:00000010:1.0:1784949387.896658:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000400:00000010:1.0:1784949387.896662:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000934e2586. 00000400:00000200:1.0:1784949387.896665:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.12@tcp 00000400:00000200:1.0:1784949387.896670:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.12@tcp local destination 00000400:00000200:1.0:1784949387.896673:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896676:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896679:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896682:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.12@tcp 00000400:00000200:1.0:1784949387.896687:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3980:3980:238:4294967295] 111.222.142.12@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:1.0:1784949387.896693:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.12@tcp 00000400:00000200:1.0:1784949387.896700:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.12@tcp(111.222.142.12@tcp:111.222.142.12@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896705:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.12@tcp 00000800:00000010:1.0:1784949387.896709:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002ddfcc2d. 00000800:00000200:1.0:1784949387.896716:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.0:1784949387.896726:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949387.896731:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ed488565] -> 12345-111.222.142.12@tcp (2) 00000800:00000200:1.1:1784949387.896757:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.12 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949387.896768:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.12@tcp 00000800:00000010:1.0:1784949387.896771:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002ddfcc2d. 00000400:00000200:1.0:1784949387.896776:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949387.896783:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.12@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949387.896787:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949387.896791:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.12@tcp: 1 00000400:00000200:1.0:1784949387.896795:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000200:1.0:1784949387.896803:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.12@tcp 00000400:00000200:1.0:1784949387.896806:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 rc 0 00000400:00000200:1.0:1784949387.896809:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 00000400:00000010:1.0:1784949387.896817:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000001df3b40e. 00000400:00000010:1.0:1784949387.896822:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b039801a. 00000400:00000010:1.0:1784949387.896826:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d4c102f3. 00000400:00000200:1.0:1784949387.896832:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.13@tcp 00000400:00000200:1.0:1784949387.896838:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.13@tcp local destination 00000400:00000200:1.0:1784949387.896842:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896846:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896849:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896854:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.13@tcp 00000400:00000200:1.0:1784949387.896859:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3981:3981:238:4294967295] 111.222.173.13@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:1.0:1784949387.896864:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.13@tcp 00000400:00000200:1.0:1784949387.896870:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.13@tcp(111.222.173.13@tcp:111.222.173.13@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.896875:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.13@tcp 00000800:00000010:1.0:1784949387.896879:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000002ddfcc2d. 00000800:00000200:1.0:1784949387.896884:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000800:00000200:1.1:1784949387.896889:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000000b333bac] -> 12345-111.222.173.13@tcp (2) 00000400:00000200:1.0:1784949387.896893:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.13@tcp 00000400:00000200:1.0:1784949387.896897:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x261 rc 0 00000400:00000200:1.0:1784949387.896902:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x2061 00000400:00000010:1.0:1784949387.896948:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000007db69e48. 00000400:00000010:1.0:1784949387.896954:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f0c9b918. 00000400:00000010:1.0:1784949387.896963:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000038468d93. 00000400:00000200:1.0:1784949387.896968:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.13@tcp 00000400:00000200:1.0:1784949387.896976:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.13@tcp local destination 00000400:00000200:1.0:1784949387.896979:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.896983:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.896987:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.896991:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.13@tcp 00000400:00000200:1.0:1784949387.896995:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3982:3982:237:4294967295] 111.222.142.13@tcp LPNI seq info [34:34:8:4294967295] 00000400:00000200:1.0:1784949387.897000:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.13@tcp 00000400:00000200:1.0:1784949387.897006:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.13@tcp(111.222.142.13@tcp:111.222.142.13@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.897019:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.13@tcp 00000800:00000010:1.0:1784949387.897025:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000004b264730. 00000800:00000200:1.0:1784949387.897031:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000800:00000200:1.1:1784949387.897039:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000051277584] -> 12345-111.222.142.13@tcp (2) 00000400:00000200:1.0:1784949387.897044:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.13@tcp 00000400:00000200:1.0:1784949387.897047:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.13@tcp(00000000181f420a) state 0x261 rc 0 00000400:00000200:1.0:1784949387.897054:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x2061 00000400:00000010:1.0:1784949387.897061:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005152bcfe. 00000400:00000010:1.0:1784949387.897065:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000079974bb9. 00000400:00000010:1.0:1784949387.897070:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008c245eeb. 00000400:00000200:1.0:1784949387.897075:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.14@tcp 00000400:00000200:1.0:1784949387.897084:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.14@tcp local destination 00000400:00000200:1.0:1784949387.897088:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.897092:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.897096:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.897100:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.14@tcp 00000400:00000200:1.0:1784949387.897104:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3983:3983:236:4294967295] 111.222.173.14@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:1.0:1784949387.897110:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.14@tcp 00000400:00000200:1.0:1784949387.897119:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.14@tcp(111.222.173.14@tcp:111.222.173.14@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.897127:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.14@tcp 00000800:00000010:1.0:1784949387.897130:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000b80ce7eb. 00000800:00000200:1.0:1784949387.897134:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000800:00000200:1.1:1784949387.897141:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000bdb14ee9] -> 12345-111.222.173.14@tcp (2) 00000400:00000200:1.0:1784949387.897159:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.14@tcp 00000400:00000200:1.0:1784949387.897164:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.14@tcp(00000000e7ed5efe) state 0x261 rc 0 00000400:00000200:1.0:1784949387.897169:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 00000400:00000010:1.0:1784949387.897174:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004215ddb7. 00000400:00000010:1.0:1784949387.897178:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000400:00000010:1.0:1784949387.897183:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000acbf6818. 00000400:00000200:1.0:1784949387.897190:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.14@tcp 00000400:00000200:1.0:1784949387.897200:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.14@tcp local destination 00000400:00000200:1.0:1784949387.897209:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.897216:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.897222:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.897228:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.14@tcp 00000400:00000200:1.0:1784949387.897236:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3984:3984:235:4294967295] 111.222.142.14@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:1.0:1784949387.897247:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.14@tcp 00000400:00000200:1.0:1784949387.897264:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.14@tcp(111.222.142.14@tcp:111.222.142.14@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.897274:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.14@tcp 00000800:00000010:1.0:1784949387.897281:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000600b3b47. 00000800:00000200:1.0:1784949387.897292:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.0:1784949387.897296:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 14 secs later) 00000800:00000200:1.1:1784949387.897303:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000308db862] -> 12345-111.222.142.14@tcp (2) 00000800:00000200:1.1:1784949387.897307:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 111.222.142.14 (cnted 0, interval 60, 14 secs later) 00000800:00000100:1.0:1784949387.897328:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-111.222.142.14@tcp 00000800:00000010:1.0:1784949387.897330:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000600b3b47. 00000400:00000200:1.0:1784949387.897335:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1784949387.897340:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.14@tcp: GET: REMOTE_ERROR 00000400:00000200:1.0:1784949387.897344:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949387.897350:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.14@tcp: 1 00000400:00000200:1.0:1784949387.897356:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000200:1.0:1784949387.897371:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.14@tcp 00000400:00000200:1.0:1784949387.897374:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 rc 0 00000400:00000200:1.0:1784949387.897377:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 00000400:00000010:1.0:1784949387.897384:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000004041a48e. 00000400:00000010:1.0:1784949387.897388:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0df5e09. 00000400:00000010:1.0:1784949387.897398:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000451e3fe4. 00000400:00000200:1.0:1784949387.897402:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.15@tcp 00000400:00000200:1.0:1784949387.897410:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.173.15@tcp local destination 00000400:00000200:1.0:1784949387.897414:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.897419:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.897433:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.897437:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.173.15@tcp 00000400:00000200:1.0:1784949387.897440:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3985:3985:235:4294967295] 111.222.173.15@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:1.0:1784949387.897445:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.15@tcp 00000400:00000200:1.0:1784949387.897454:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.15@tcp(111.222.173.15@tcp:111.222.173.15@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.897460:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.15@tcp 00000800:00000010:1.0:1784949387.897465:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000600b3b47. 00000800:00000200:1.0:1784949387.897470:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000800:00000200:1.1:1784949387.897478:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b45c29f5] -> 12345-111.222.173.15@tcp (2) 00000400:00000200:1.0:1784949387.897483:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.15@tcp 00000400:00000200:1.0:1784949387.897486:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x261 rc 0 00000400:00000200:1.0:1784949387.897490:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x2061 00000400:00000010:1.0:1784949387.897497:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000002c1a3d41. 00000400:00000010:1.0:1784949387.897500:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a2727e1c. 00000400:00000010:1.0:1784949387.897506:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000009234a5fd. 00000400:00000200:1.0:1784949387.897510:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.142.15@tcp 00000400:00000200:1.0:1784949387.897514:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 111.222.142.15@tcp local destination 00000400:00000200:1.0:1784949387.897518:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1784949387.897521:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.897524:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1784949387.897528:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 111.222.142.15@tcp 00000400:00000200:1.0:1784949387.897534:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3986:3986:234:4294967295] 111.222.142.15@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:1.0:1784949387.897540:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.142.15@tcp 00000400:00000200:1.0:1784949387.897547:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.142.15@tcp(111.222.142.15@tcp:111.222.142.15@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.897552:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.142.15@tcp 00000800:00000010:1.0:1784949387.897555:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000be1f3156. 00000800:00000200:1.0:1784949387.897559:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000800:00000200:1.1:1784949387.897562:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000113471b7] -> 12345-111.222.142.15@tcp (2) 00000400:00000200:1.0:1784949387.897566:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.142.15@tcp 00000400:00000200:1.0:1784949387.897568:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.15@tcp(00000000478f043b) state 0x261 rc 0 00000400:00000200:1.0:1784949387.897572:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.31@tcp(00000000bf393d23) state 0x40 00000400:00000010:1.0:1784949387.897575:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000047e19e91. 00000400:00000010:1.0:1784949387.897578:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a01227d5. 00000400:00000010:1.0:1784949387.897581:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000003859b1c5. 00000400:00000200:1.0:1784949387.897584:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-111.222.173.31@tcp 00000400:00000200:1.0:1784949387.897588:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 111.222.173.31@tcp local destination 00000400:00000200:1.0:1784949387.897591:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:1.0:1784949387.897596:0:337117:0:(lib-move.c:1874:lnet_set_non_mr_pref_nid()) Setting preferred local NID 192.168.202.137@tcp on NMR peer 111.222.173.31@tcp 00000400:00000200:1.0:1784949387.897601:0:337117:0:(peer.c:1086:lnet_peer_ni_set_non_mr_pref_nid()) peer 111.222.173.31@tcp nid 192.168.202.137@tcp: 0 00000400:00000200:1.0:1784949387.897606:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3987:3987:233:4294967295] 111.222.173.31@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:1.0:1784949387.897611:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 111.222.173.31@tcp 00000400:00000200:1.0:1784949387.897620:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 111.222.173.31@tcp(111.222.173.31@tcp:111.222.173.31@tcp) : GET try# 0 00000800:00000200:1.0:1784949387.897626:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-111.222.173.31@tcp 00000800:00000010:1.0:1784949387.897630:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000a1bdc144. 00000800:00000010:1.0:1784949387.897639:0:337117:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 0000000071784be7. 00000800:00000010:1.0:1784949387.897645:0:337117:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000d60fdb75. 00000800:00000200:1.0:1784949387.897653:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000071784be7] -> 12345-111.222.173.31@tcp (2) 00000800:00000200:1.1:1784949387.897659:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000071784be7] -> 12345-111.222.173.31@tcp (2) 00000400:00000200:1.0:1784949387.897671:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 111.222.173.31@tcp 00000400:00000200:1.0:1784949387.897676:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.31@tcp(00000000bf393d23) state 0x240 rc 0 00000400:00000200:1.0:1784949387.897683:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x861 00000400:00000001:1.0:1784949387.897686:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.897689:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.897692:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e5294b24 00000400:00000010:1.0:1784949387.897695:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e5294b24. 00000400:00000200:1.0:1784949387.897699:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.897703:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.4@tcp: -114 00000400:00000010:1.0:1784949387.897716:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000307ddfde. 00000400:00000200:1.0:1784949387.897722:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.4@tcp:-5 00000400:00000200:1.0:1784949387.897728:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.4@tcp(00000000872a6639) state 0x2061 rc -5 00000400:00000200:1.0:1784949387.897756:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.4@tcp 00000400:00000200:1.0:1784949387.897766:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x861 00000400:00000001:1.0:1784949387.897771:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.897773:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.897775:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d777f653 00000400:00000010:1.0:1784949387.897777:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000d777f653. 00000400:00000200:1.0:1784949387.897781:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.897784:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.6@tcp: -114 00000400:00000010:1.0:1784949387.897787:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000038ac69a2. 00000400:00000200:1.0:1784949387.897794:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.6@tcp:-5 00000400:00000200:1.0:1784949387.897797:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.6@tcp(00000000575da65d) state 0x2061 rc -5 00000400:00000200:1.0:1784949387.897801:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.6@tcp 00000400:00000200:1.0:1784949387.897805:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x861 00000400:00000001:1.0:1784949387.897808:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.897809:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.897811:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ed151fd1 00000400:00000010:1.0:1784949387.897812:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ed151fd1. 00000400:00000200:1.0:1784949387.897816:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.897824:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.8@tcp: -114 00000400:00000010:1.0:1784949387.897827:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000053ab16eb. 00000400:00000200:1.0:1784949387.897832:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.8@tcp:-5 00000400:00000200:1.0:1784949387.897837:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.8@tcp(00000000be97fe94) state 0x2061 rc -5 00000400:00000200:1.0:1784949387.897845:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.8@tcp 00000400:00000200:1.0:1784949387.897853:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x861 00000400:00000001:1.0:1784949387.897856:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.897858:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.897860:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000400:00000010:1.0:1784949387.897863:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00000400:00000200:1.0:1784949387.897866:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.897870:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.10@tcp: -114 00000400:00000010:1.0:1784949387.897872:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000f6f633a2. 00000400:00000200:1.0:1784949387.897876:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.10@tcp:-5 00000400:00000200:1.0:1784949387.897878:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.10@tcp(00000000ba75d963) state 0x2061 rc -5 00000400:00000200:1.0:1784949387.897881:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.10@tcp 00000400:00000200:1.0:1784949387.897889:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x861 00000400:00000001:1.0:1784949387.897892:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.897894:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.897896:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:1.0:1784949387.897897:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000400:00000200:1.0:1784949387.897902:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.897945:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.12@tcp: -114 00000400:00000010:1.0:1784949387.897950:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000000f0e4e38. 00000400:00000200:1.0:1784949387.897957:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.12@tcp:-5 00000400:00000200:1.0:1784949387.897961:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.12@tcp(000000003e35c3d8) state 0x2061 rc -5 00000400:00000200:1.0:1784949387.897965:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.12@tcp 00000400:00000200:1.0:1784949387.897970:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x861 00000400:00000001:1.0:1784949387.897974:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949387.897976:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949387.897978:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000400:00000010:1.0:1784949387.897980:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000400:00000200:1.0:1784949387.897983:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949387.897988:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.14@tcp: -114 00000400:00000010:1.0:1784949387.897992:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004215ddb7. 00000400:00000200:1.0:1784949387.897995:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.14@tcp:-5 00000400:00000200:1.0:1784949387.898001:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.14@tcp(000000001441465b) state 0x2061 rc -5 00000400:00000200:1.0:1784949387.898005:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.14@tcp 00010000:00000001:2.0:1784949388.532031:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949388.532040:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949388.532049:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949388.532051:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949388.532197:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949388.532204:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949388.532209:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949388.724043:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949388.724061:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949388.724064:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949388.724072:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949388.724076:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949388.724081:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949388.724085:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1784949388.724085:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949388.724100:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949388.724115:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ca646268. 00000020:00000040:0.0:1784949388.724127:0:89766:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:0.0:1784949388.724136:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949388.724142:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949388.724150:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949388.724163:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000662152ce. 00000100:00000001:0.0:1784949388.724175:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949388.724189:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949388.724212:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949388.724217:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949388.724227:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca646268] to pc [ptlrpcd_00_01+1] req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949388.724279:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949388.724285:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949388.724304:0:89766:0:(pinger.c:247:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1784949388.724309:0:89766:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1784949388.724315:0:89766:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949388.724322:0:89766:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bab47beb. 00000020:00000040:0.0:1784949388.724328:0:89766:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:0.0:1784949388.724332:0:89766:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1784949388.724335:0:89766:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949388.724340:0:89766:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949388.724347:0:89766:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 00000000820f02b3. 00000100:00000001:0.0:1784949388.724354:0:89766:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949388.724366:0:89766:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949388.724370:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949388.724379:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949388.724385:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949388.724395:0:89766:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1784949388.724397:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949388.724402:0:89766:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949388.724407:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949388.724417:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949388.724423:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784949388.724426:0:89766:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bab47beb] to pc [ptlrpcd_00_02+2] req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.724428:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949388.724437:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1784949388.724443:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1784949388.724452:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1784949388.724463:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289693824:0@lo:400: 00000100:00000001:1.0:1784949388.724499:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1784949388.724508:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949388.724511:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949388.724513:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1784949388.724520:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004215ddb7. 02000000:00000001:1.0:1784949388.724522:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949388.724530:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000001:0.0:1784949388.724537:0:89766:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949388.724539:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000100:00000040:0.0:1784949388.724552:0:89766:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11665) 00000100:00000200:1.0:1784949388.724553:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693824, portal 25 00000100:00000001:3.0:1784949388.724559:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949388.724561:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1784949388.724565:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469564544 : -115544239987072 : ffff96e9c29ea680) 00000100:00000001:3.0:1784949388.724566:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949388.724571:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949388.724573:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949388.724582:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.724590:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1784949388.724600:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:1.0:1784949388.724607:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000100:00000001:3.0:1784949388.724609:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000200:1.0:1784949388.724613:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289693824, offset 0 00000100:00000001:3.0:1784949388.724614:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949388.724623:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1784949388.724627:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 02000000:00000001:3.0:1784949388.724628:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949388.724630:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949388.724637:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949388.724638:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949388.724646:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289693952:192.168.202.137@tcp:400: 00000400:00000200:1.0:1784949388.724651:0:337120:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949388.724655:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1784949388.724661:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1784949388.724663:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949388.724667:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1784949388.724672:0:337120:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643fe80 02000000:00000010:3.0:1784949388.724675:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004565e281. 02000000:00000001:3.0:1784949388.724680:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949388.724683:0:337120:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8edd [2] + 0 00000400:00000010:3.0:1784949388.724688:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000001004a12d. 00000400:00000010:1.0:1784949388.724690:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000789ee428. 00000400:00000200:1.0:1784949388.724710:0:337120:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000004dd24a2e 00000400:00000010:3.0:1784949388.724717:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bb55de7f. 00000400:00000200:1.0:1784949388.724720:0:337120:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:3.0:1784949388.724726:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289693952, portal 10 00000400:00000200:1.0:1784949388.724749:0:337120:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000400:00000200:1.0:1784949388.724756:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004dd24a2e 00000400:00000010:1.0:1784949388.724761:0:337120:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000004dd24a2e. 00000100:00000001:3.0:1784949388.724763:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949388.724767:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469556480 : -115544239995136 : ffff96e9c29e8700) 00000100:00000001:1.0:1784949388.724770:0:337120:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949388.724775:0:337120:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000040:3.0:1784949388.724778:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949388.724782:0:337120:0:(events.c:337:request_in_callback()) incoming req@000000003b52fa03 x1871651289693824 msgsize 224 00000100:00100000:1.0:1784949388.724799:0:337120:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000040:1.0:1784949388.724806:0:337120:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1784949388.724819:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949388.724823:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949388.724825:0:337120:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000010:3.0:1784949388.724826:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0e1f1a0. 00000100:00000200:3.0:1784949388.724832:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289693952, offset 0 00000400:00000200:1.0:1784949388.724837:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000400:00000010:1.0:1784949388.724841:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000100:00000001:1.0:1784949388.724846:0:337120:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1784949388.724852:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000200:1.0:1784949388.724853:0:337120:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.724868:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1784949388.724869:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000400:00000200:3.0:1784949388.724872:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00100000:2.0:1784949388.724875:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693824 00000100:00000040:1.0:1784949388.724876:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1784949388.724883:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000400:00000200:3.0:1784949388.724887:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784949388.724889:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949388.724896:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949388.724900:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:2.0:1784949388.724933:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949388.724939:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693824 00000020:00000001:2.0:1784949388.724943:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1784949388.724945:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000100:00000001:1.0:1784949388.724947:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949388.724949:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1784949388.724955:0:337120:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:2.0:1784949388.724956:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000020:00000001:2.0:1784949388.724964:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:3.0:1784949388.724964:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1784949388.724970:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:3.0:1784949388.724974:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000001:2.0:1784949388.724978:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:3.0:1784949388.725001:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3988:3988:232:4294967295] 192.168.202.137@tcp LPNI seq info [2538:2538:8:4294967295] 00000020:00000010:2.0:1784949388.725005:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000001:2.0:1784949388.725013:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1784949388.725020:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000400:00000200:3.0:1784949388.725029:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000010:2.0:1784949388.725035:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c405a384. 00000800:00000200:3.0:1784949388.725044:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949388.725045:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949388.725047:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b49ef65d. 00000800:00000010:3.0:1784949388.725057:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000100:00000001:1.0:1784949388.725059:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306328107 00000020:00000010:2.0:1784949388.725061:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000521c0956. 00000100:00000010:1.0:1784949388.725074:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000800:00000200:3.0:1784949388.725075:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:2.0:1784949388.725079:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949388.725089:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949388.725093:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1784949388.725093:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000800:00000200:3.0:1784949388.725102:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000100:00000001:2.0:1784949388.725103:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949388.725114:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1784949388.725116:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306328107 00000100:00000001:2.0:1784949388.725120:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949388.725125:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d583450. 00000100:00000001:1.0:1784949388.725132:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1784949388.725136:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306328107 00000100:00000001:3.0:1784949388.725140:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949388.725141:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1784949388.725142:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000070432f3d. 00000100:00000040:3.0:1784949388.725155:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2263 usecs (3 jiffies) 00000100:00000001:1.0:1784949388.725157:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1784949388.725167:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306328068 00000100:00000001:1.0:1784949388.725173:0:337120:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306328107 00000100:00000001:3.0:1784949388.725175:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306328107 00000100:00000001:2.0:1784949388.725177:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000010:1.0:1784949388.725184:0:337120:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000006a83a85b. 00000100:00000001:2.0:1784949388.725187:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949388.725188:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306328068 00000100:00000001:1.0:1784949388.725196:0:337120:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1784949388.725198:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306328107 00000100:00100000:2.0:1784949388.725206:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 842 00000100:00000001:3.0:1784949388.725209:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306328068 00000100:00000001:1.0:1784949388.725211:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949388.725215:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:3.0:1784949388.725223:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306328107 00000100:00000001:1.0:1784949388.725224:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949388.725225:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782157904 : -115538927393712 : ffff96eaff466450) 00000100:00000001:3.0:1784949388.725232:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306328068 00000100:00000001:1.0:1784949388.725237:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949388.725240:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306328107 00000100:00000040:2.0:1784949388.725245:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000003b52fa03 x1871651289693824/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:294/0 lens 224/0 e 0 to 0 dl 1784949399 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.725247:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949388.725250:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949388.725263:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949388.725264:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949388.725272:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1784949388.725273:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:3.0:1784949388.725275:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949388.725279:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949388.725282:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784949388.725283:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 0000000076268fb0. 00000100:00000001:1.0:1784949388.725285:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949388.725293:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949388.725294:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949388.725295:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949388.725302:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949388.725303:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000020:00000010:2.0:1784949388.725304:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 0000000076268fb0. 00000100:00000001:3.0:1784949388.725317:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949388.725324:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1784949388.725325:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000003b52fa03 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289693824:12345-0@lo:400: 00000800:00000200:0.0:1784949388.725330:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1784949388.725333:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949388.725338:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693824 00000100:00000001:3.0:1784949388.725344:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1784949388.725345:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000020:00000001:2.0:1784949388.725346:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:2.0:1784949388.725354:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:2.0:1784949388.725359:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949388.725363:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:2.0:1784949388.725363:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000001:1.0:1784949388.725367:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949388.725369:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000020:00000001:2.0:1784949388.725378:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:2.0:1784949388.725383:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949388.725388:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:1.0:1784949388.725389:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949388.725390:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949388.725399:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000800:00000001:1.0:1784949388.725400:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:2.0:1784949388.725407:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949388.725408:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0e1f1a0 00000800:00000001:1.0:1784949388.725410:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949388.725414:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000400:00000010:0.0:1784949388.725420:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0e1f1a0. 00000100:00000001:2.0:1784949388.725422:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:2.0:1784949388.725431:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949388.725436:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1784949388.725436:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 02000000:00000010:2.0:1784949388.725439:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000077ad1613. 02000000:00000001:2.0:1784949388.725449:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1784949388.725454:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949388.725456:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949388.725482:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c643ff00 00000020:00000001:2.0:1784949388.725490:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949388.725500:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693824 00000400:00000200:1.0:1784949388.725508:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 184272 00010000:00000001:2.0:1784949388.725509:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949388.725515:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949388.725520:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00010000:00000200:2.0:1784949388.725521:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000003b52fa03 x1871651289693824/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:294/0 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949388.725527:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949388.725530:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949388.725538:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949388.725550:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00010000:00000001:2.0:1784949388.725558:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949388.725562:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949388.725563:0:337112:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949388.725568:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00001000:2.0:1784949388.725570:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=110 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949388.725574:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1784949388.725577:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007a1d0aba. 00000100:00000001:2.0:1784949388.725580:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1784949388.725583:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1784949388.725584:0:337112:0:(events.c:337:request_in_callback()) incoming req@000000007a1d0aba x1871651289693952 msgsize 224 00000100:00000040:2.0:1784949388.725588:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949388.725594:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000100:00100000:1.0:1784949388.725600:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:2.0:1784949388.725602:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:2.0:1784949388.725607:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949388.725612:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949388.725616:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949388.725621:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000100:00000001:1.0:1784949388.725622:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000200:2.0:1784949388.725630:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289693824, offset 224 00000800:00000001:1.0:1784949388.725635:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949388.725639:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000001:1.0:1784949388.725644:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949388.725654:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:2.0:1784949388.725667:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643fe80 00000400:00000200:2.0:1784949388.725674:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x98ed [1] + 224 00000100:00000001:3.0:1784949388.725675:0:343544:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:3.0:1784949388.725681:0:343544:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289693952 00000400:00000200:2.0:1784949388.725685:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1784949388.725686:0:343544:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:3.0:1784949388.725692:0:343544:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000200:2.0:1784949388.725696:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949388.725698:0:343544:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949388.725702:0:342756:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1784949388.725704:0:343544:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949388.725716:0:343544:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289693952 00000100:00000200:2.0:1784949388.725719:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949388.725720:0:343544:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949388.725725:0:343544:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:3.0:1784949388.725744:0:343544:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1784949388.725747:0:343544:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000020:00000001:3.0:1784949388.725754:0:343544:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949388.725762:0:343544:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000001:3.0:1784949388.725768:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000040:2.0:1784949388.725769:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1784949388.725775:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001e9e3d4b. 00000100:00000001:3.0:1784949388.725782:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949388.725789:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001e9e3d4b. 00000020:00000010:3.0:1784949388.725799:0:343544:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005cc3cd5d. 00000100:00000001:2.0:1784949388.725803:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1784949388.725804:0:343544:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000926a9e9b. 00000020:00000010:3.0:1784949388.725811:0:343544:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000200:2.0:1784949388.725813:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:2.0:1784949388.725817:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000100:00000040:3.0:1784949388.725819:0:343544:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:2.0:1784949388.725821:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949388.725826:0:343544:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:2.0:1784949388.725827:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:3.0:1784949388.725833:0:343544:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:2.0:1784949388.725837:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949388.725838:0:343544:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.725844:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1784949388.725848:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00000001:1.0:1784949388.725850:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949388.725850:0:343544:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00010000:00000001:2.0:1784949388.725856:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000001:3.0:1784949388.725857:0:343544:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000001:2.0:1784949388.725861:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949388.725862:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1784949388.725862:0:343544:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1784949388.725868:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1784949388.725870:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000400:00000010:1.0:1784949388.725873:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000100:00000040:2.0:1784949388.725878:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003b52fa03 x1871651289693824/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:294/0 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949388.725881:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000100:00000001:3.0:1784949388.725886:0:343544:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1784949388.725887:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000100:00000001:1.0:1784949388.725898:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1784949388.725899:0:343544:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:2.0:1784949388.725902:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000003b52fa03 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337120:x1871651289693824:12345-0@lo:400: Request processed in 626us (1116us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949388.725942:0:343544:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:1.0:1784949388.725947:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949388.725951:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 842 00000100:00100000:3.0:1784949388.725956:0:343544:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 818 00000100:00000040:2.0:1784949388.725963:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000040:3.0:1784949388.725966:0:343544:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:2.0:1784949388.725971:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949388.725974:0:343544:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554893312 : -115544154658304 : ffff96e9c7b4aa00) 00000100:00000001:2.0:1784949388.725977:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:3.0:1784949388.726014:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000007a1d0aba x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:294/0 lens 224/0 e 0 to 0 dl 1784949399 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949388.726016:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b49ef65d. 00000100:00000200:1.0:1784949388.726020:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1784949388.726028:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000521c0956. 00000020:00000010:2.0:1784949388.726039:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c405a384. 00000020:00000040:2.0:1784949388.726052:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 02000000:00000001:2.0:1784949388.726064:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 02000000:00000010:2.0:1784949388.726077:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000077ad1613. 00000100:00000001:3.0:1784949388.726086:0:343544:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:1.0:1784949388.726087:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1784949388.726088:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000020:00000010:3.0:1784949388.726096:0:343544:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000001e9e3d4b. 00000100:00000001:2.0:1784949388.726098:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.726099:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949388.726108:0:343544:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949388.726112:0:337120:0:(client.c:1580:after_reply()) Process entered 00000100:00000200:2.0:1784949388.726113:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000020:00000010:3.0:1784949388.726114:0:343544:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000001e9e3d4b. 02000000:00000001:1.0:1784949388.726118:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949388.726125:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000400:00000010:2.0:1784949388.726128:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000054d5ba64. 00000100:00000001:1.0:1784949388.726129:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1784949388.726134:0:343544:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000007a1d0aba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289693952:12345-192.168.202.137@tcp:400: 02000000:00000001:1.0:1784949388.726140:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949388.726143:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c1d3a6d6. 00000100:00000200:3.0:1784949388.726149:0:343544:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289693952 00000100:00001000:1.0:1784949388.726154:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=110 v=5 (5 5 5 5) 00000020:00000001:3.0:1784949388.726156:0:343544:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:3.0:1784949388.726164:0:343544:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000100:00001000:1.0:1784949388.726166:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=110 v=5 (1 1 1 1) 00000020:00000001:3.0:1784949388.726171:0:343544:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949388.726174:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000020:00000001:3.0:1784949388.726177:0:343544:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:1.0:1784949388.726180:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949388.726182:0:343544:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00010000:00000001:1.0:1784949388.726187:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000020:00000001:3.0:1784949388.726190:0:343544:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00010000:00000001:1.0:1784949388.726193:0:337120:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949388.726197:0:343544:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.726199:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949388.726205:0:343544:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000040:1.0:1784949388.726211:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949388.726213:0:343544:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949388.726219:0:343544:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949388.726227:0:343544:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949388.726235:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1784949388.726235:0:343544:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:3.0:1784949388.726241:0:343544:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:1.0:1784949388.726243:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949388.726248:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1784949388.726251:0:343544:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 000000000239f355. 00000100:00000040:1.0:1784949388.726253:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1784949388.726261:0:343544:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949388.726270:0:343544:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949388.726279:0:343544:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949388.726287:0:343544:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289693952 00000100:00100000:1.0:1784949388.726291:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000ca646268 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337120:1871651289693824:0@lo:400: 00010000:00000001:3.0:1784949388.726296:0:343544:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:1.0:1784949388.726304:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00010000:00000200:3.0:1784949388.726310:0:343544:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000007a1d0aba x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:294/0 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.726312:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949388.726321:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000ca646268 x1871651289693824/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949388.726353:0:343544:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00000100:00000001:1.0:1784949388.726357:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00010000:00000001:3.0:1784949388.726359:0:343544:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949388.726363:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000100:00001000:3.0:1784949388.726366:0:343544:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=105 v=5 (1 1 1 1) 02000000:00000010:1.0:1784949388.726370:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004215ddb7. 00000100:00000001:3.0:1784949388.726375:0:343544:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949388.726378:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1784949388.726384:0:343544:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000020:00000001:1.0:1784949388.726386:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1784949388.726392:0:343544:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000040:1.0:1784949388.726393:0:337120:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 02000000:00000001:3.0:1784949388.726403:0:343544:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000020:00000001:1.0:1784949388.726405:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:3.0:1784949388.726409:0:343544:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1784949388.726411:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000662152ce. 00000100:00000001:3.0:1784949388.726421:0:343544:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1784949388.726423:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949388.726428:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:3.0:1784949388.726430:0:343544:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000010:1.0:1784949388.726432:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ca646268. 00000400:00000010:3.0:1784949388.726437:0:343544:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b013cebf. 00000100:00000001:1.0:1784949388.726440:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949388.726445:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949388.726447:0:343544:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289693952, offset 224 00000100:00000001:1.0:1784949388.726449:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.726460:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949388.726483:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949388.726484:0:343544:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949388.726489:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949388.726494:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949388.726498:0:343544:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949388.726506:0:343544:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3989:3989:232:4294967295] 192.168.202.137@tcp LPNI seq info [2539:2539:8:4294967295] 00000400:00000200:3.0:1784949388.726515:0:343544:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949388.726521:0:343544:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949388.726524:0:343544:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005f459fde. 00000800:00000200:3.0:1784949388.726529:0:343544:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949388.726534:0:343544:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949388.726537:0:343544:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005f459fde type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949388.726552:0:343544:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949388.726555:0:343544:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949388.726557:0:343544:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949388.726559:0:343544:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949388.726561:0:343544:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949388.726564:0:343544:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007a1d0aba x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:294/0 lens 224/224 e 0 to 0 dl 1784949399 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949388.726573:0:343544:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000007a1d0aba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289693952:12345-192.168.202.137@tcp:400: Request processed in 489us (988us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949388.726581:0:343544:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 818 00000100:00000040:3.0:1784949388.726584:0:343544:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949388.726587:0:343544:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949388.726589:0:343544:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949388.726592:0:343544:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000926a9e9b. 00000020:00000010:3.0:1784949388.726597:0:343544:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949388.726601:0:343544:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005cc3cd5d. 00000020:00000040:3.0:1784949388.726607:0:343544:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949388.726610:0:343544:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949388.726816:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:3.0:1784949388.726822:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000010:0.0:1784949388.726823:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005f459fde. 00000400:00000200:0.0:1784949388.726833:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:3.0:1784949388.726833:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949388.726839:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949388.726844:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949388.726851:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949388.726858:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:0.0:1784949388.726864:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b013cebf 00000400:00000200:3.0:1784949388.726871:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c643ff00 00000400:00000010:0.0:1784949388.726872:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b013cebf. 00000100:00000001:0.0:1784949388.726882:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:0.0:1784949388.726885:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000400:00000200:3.0:1784949388.726890:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x98f5 [1] + 224 00000800:00000001:3.0:1784949388.726898:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949388.726933:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949388.726936:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949388.726940:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949388.726945:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949388.726949:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949388.726960:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949388.726993:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949388.727001:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949388.727005:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.727097:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949388.727103:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1784949388.727110:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949388.727112:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1784949388.727115:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000001004a12d. 00000400:00000200:1.0:1784949388.727125:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bb55de7f 00000400:00000010:1.0:1784949388.727127:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000bb55de7f. 00000100:00000001:1.0:1784949388.727131:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1784949388.727138:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1784949388.727151:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.727161:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1784949388.727163:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.727166:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:1.0:1784949388.727168:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1784949388.727171:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1784949388.727172:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1784949388.727175:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1784949388.727181:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=105 v=5 (5 5 5 5) 00000100:00001000:1.0:1784949388.727185:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=105 v=5 (1 1 1 1) 00000100:00000001:1.0:1784949388.727189:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1784949388.727191:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1784949388.727193:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1784949388.727197:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1784949388.727207:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949388.727210:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1784949388.727212:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:1.0:1784949388.727214:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1784949388.727216:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949388.727219:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.727228:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949388.727230:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949388.727232:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1784949388.727236:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1784949388.727244:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000bab47beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289693952:192.168.202.137@tcp:400: 00000100:00000001:1.0:1784949388.727249:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.727251:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1784949388.727253:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000bab47beb x1871651289693952/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949388.727261:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1784949388.727263:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1784949388.727265:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004565e281. 02000000:00000001:1.0:1784949388.727271:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1784949388.727272:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1784949388.727274:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:1.0:1784949388.727277:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1784949388.727279:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 00000000820f02b3. 02000000:00000001:1.0:1784949388.727282:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1784949388.727283:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1784949388.727286:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000bab47beb. 00000100:00000001:1.0:1784949388.727290:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1784949388.727292:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.727294:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949388.727299:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1784949388.727300:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949388.727303:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949388.727305:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949388.768023:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306328107, at 4306328108 00000100:00000040:1.0:1784949388.768039:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306328107, at 4306328108 00000100:00000010:2.0:1784949388.768039:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000070432f3d. 00000100:00000040:0.0:1784949388.768045:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306328107, at 4306328108 00000100:00000040:3.0:1784949388.768045:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306328107, at 4306328108 00000100:00000010:1.0:1784949388.768056:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d583450. 00000100:00000010:3.0:1784949388.768063:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000006a83a85b. 00000100:00000010:0.0:1784949388.768068:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000800:00000100:3.0:1784949388.916112:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.142.5@tcp 00000800:00000010:3.0:1784949388.916119:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000061dbde97. 00000400:00000200:3.0:1784949388.916130:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949388.916141:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.142.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949388.916151:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1784949388.916157:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.142.5@tcp: 1 00000400:00000200:3.0:1784949388.916166:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000800:00000100:3.0:1784949388.916200:0:337111:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.11@tcp 00000800:00000010:3.0:1784949388.916204:0:337111:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000017e5e4f6. 00000400:00000200:3.0:1784949388.916209:0:337111:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:3.0:1784949388.916217:0:337111:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.11@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949388.916224:0:337111:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949388.916227:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000200:3.0:1784949388.916234:0:337111:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.11@tcp: 1 00000400:00000010:2.0:1784949388.916240:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 0000000079a52f3b. 00000400:00000200:3.0:1784949388.916241:0:337111:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000200:1.0:1784949388.916258:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:2.0:1784949388.916261:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000010:2.0:1784949388.916271:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000addcc8e4. 00000400:00000200:1.0:1784949388.916281:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2861 00000400:00000010:2.0:1784949388.916282:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 0000000008fa7414. 00000400:00000200:2.0:1784949388.916297:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000001:1.0:1784949388.916307:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949388.916312:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949388.916315:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006fb22a7b 00000400:00000010:1.0:1784949388.916319:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000006fb22a7b. 00000400:00000200:1.0:1784949388.916326:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949388.916333:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.142.5@tcp: -114 00000400:00000010:1.0:1784949388.916338:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000003cdea007. 00000400:00000200:1.0:1784949388.916344:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.142.5@tcp:-110 00000400:00000200:1.0:1784949388.916348:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.142.5@tcp(000000008058bc02) state 0x2061 rc -110 00000400:00000200:2.0:1784949388.916361:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949388.916373:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949388.916379:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3990:3990:234:4294967295] 192.168.202.137@tcp LPNI seq info [2540:2540:8:4294967295] 00000400:00000200:2.0:1784949388.916391:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:2.0:1784949388.916404:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000400:00000200:1.0:1784949388.916423:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.142.5@tcp 00000800:00000200:2.0:1784949388.916424:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949388.916440:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f64c9b3a. 00000400:00000200:1.0:1784949388.916444:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2861 00000800:00000200:2.0:1784949388.916456:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000400:00000001:1.0:1784949388.916457:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949388.916466:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949388.916472:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000357876c0 00000800:00000200:2.0:1784949388.916478:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000010:1.0:1784949388.916481:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000357876c0. 00000800:00000200:2.0:1784949388.916488:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000f64c9b3a type 2, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1784949388.916493:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949388.916507:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.11@tcp: -114 00000400:00000010:1.0:1784949388.916514:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000cdf2ee22. 00000400:00000200:1.0:1784949388.916521:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.11@tcp:-110 00000400:00000200:1.0:1784949388.916526:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.11@tcp(0000000003351dc0) state 0x2061 rc -110 00000400:00000200:2.0:1784949388.916535:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11662 health :800 00000400:00000200:1.0:1784949388.916549:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.11@tcp 00000800:00000200:3.0:1784949388.916646:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949388.916652:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f64c9b3a. 00000400:00000200:3.0:1784949388.916658:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949388.916663:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:3.0:1784949388.916668:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:3.0:1784949388.916672:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949388.916684:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949388.916695:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949388.916699:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949388.916704:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949388.916721:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949388.916759:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949388.916770:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949388.916780:0:337112:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949388.916786:0:337112:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3991:3991:234:4294967295] 192.168.202.137@tcp LPNI seq info [2541:2541:8:4294967295] 00000400:00000200:3.0:1784949388.916797:0:337112:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949388.916803:0:337112:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949388.916808:0:337112:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949388.916814:0:337112:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949388.916821:0:337112:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949388.916825:0:337112:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 3, nob 144 niov 1 nkiov 1 00000800:00000001:0.0:1784949388.916889:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949388.916981:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:3.0:1784949388.916990:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1784949388.916994:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1784949388.916996:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000800:00000001:0.0:1784949388.917003:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:3.0:1784949388.917005:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000001:0.0:1784949388.917011:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949388.917016:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949388.917027:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:0.0:1784949388.917050:0:337113:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9909 00000800:00000001:0.0:1784949388.917059:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1784949388.917070:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1784949388.917073:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1784949388.917082:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949388.917088:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:0.0:1784949388.917095:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000addcc8e4 00000400:00000010:0.0:1784949388.917099:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000addcc8e4. 00000400:00000200:0.0:1784949388.917104:0:337113:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:0.0:1784949388.917108:0:337113:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 0000000079a52f3b. 00000400:00000010:0.0:1784949388.917113:0:337113:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000800:00000001:0.0:1784949388.917122:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1784949388.917125:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1784949389.556052:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949389.556066:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949389.556078:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949389.556081:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949389.556211:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949389.556218:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949389.556223:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949389.748130:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949389.748164:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949389.748167:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949389.748173:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949389.748177:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949389.748182:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949389.748186:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949390.580024:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949390.580034:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949390.580043:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949390.580045:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949390.580122:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949390.580128:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949390.580133:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949390.772084:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949390.772101:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949390.772105:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949390.772113:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949390.772116:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949390.772122:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949390.772126:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949391.604028:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949391.604038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949391.604046:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949391.604048:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949391.604171:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949391.604177:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949391.604182:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1784949391.796113:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1784949391.796126:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949391.796129:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949391.796134:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949391.796137:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949391.796142:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949391.796146:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949392.628029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949392.628038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949392.628047:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949392.628051:0:343945:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1784949392.628055:0:343945:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1784949392.628058:0:343945:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949392.628062:0:343945:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1784949392.628065:0:343945:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949392.628067:0:343945:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949392.628070:0:343945:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949392.628199:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949392.628206:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949392.628210:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949392.820090:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949392.820109:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949392.820112:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949392.820119:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949392.820123:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949392.820129:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949392.820132:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949393.652060:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949393.652070:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949393.652081:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949393.652085:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949393.652155:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949393.652161:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949393.652165:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949393.844026:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949393.844034:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1784949393.844045:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1784949393.844052:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1784949368/real 1784949368] req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949393 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949393.844060:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1784949393.844075:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b83ea808. 00000400:00000001:2.0:1784949393.844082:0:337118:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:0.0:1784949393.844089:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000400:00000001:2.0:1784949393.844089:0:337118:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1784949393.844094:0:337118:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000047084e78. 00000100:00000001:0.0:1784949393.844103:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1784949393.844107:0:337118:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000000226c39 02000000:00000001:0.0:1784949393.844113:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1784949393.844113:0:337118:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000000226c39. 00000100:00000001:2.0:1784949393.844124:0:337118:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1784949393.844127:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949393.844128:0:337118:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949393 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949393.844145:0:337118:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949393 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949393.844148:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002a753d9d. 00000100:00000001:2.0:1784949393.844162:0:337118:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949393.844165:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.844166:0:337118:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949393.844179:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949393.844181:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949393.844188:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1784949393.844189:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949393 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949393.844254:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949393.844269:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949393.844274:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.844281:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949393.844284:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1784949393.844285:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949393.844293:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.844293:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949393.844308:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949393 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949393.844315:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b83ea808] to pc [ptlrpcd_00_03+3] req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949393.844379:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949393.844388:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949393.844390:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949393.844391:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949393.844395:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949393.844397:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949393.844401:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1784949393.844423:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1784949393.844428:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.137@tcp failed (-110) 00000100:00080000:0.0:1784949393.844434:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00000001:2.0:1784949393.844436:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949393.844449:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949393 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1784949393.844450:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949393.844460:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949393.844468:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1784949393.844473:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003dd0d971 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289692800:192.168.202.137@tcp:8: 00000100:00000001:1.0:1784949393.844474:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949393.844480:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1784949393.844489:0:337121:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00000001:2.0:1784949393.844491:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1784949393.844498:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11640 00000100:00000001:2.0:1784949393.844504:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1784949393.844505:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949393.844516:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003dd0d971 x1871651289692800/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949393 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949393.844522:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1784949393.844525:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1784949393.844526:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:1.0:1784949393.844531:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949393.844536:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949393.844546:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11639 00000100:00000001:1.0:1784949393.844548:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949393.844558:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949393.844559:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1784949393.844568:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1784949393.844584:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949393.844590:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000400:00000200:3.0:1784949393.844607:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1784949393.844608:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000400:00000200:3.0:1784949393.844623:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00000001:1.0:1784949393.844626:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1784949393.844630:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:1.0:1784949393.844634:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.844641:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949393.844647:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1784949393.844653:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 02000000:00000001:1.0:1784949393.844657:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949393.844660:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 00000400:00000010:3.0:1784949393.844663:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 02000000:00000001:1.0:1784949393.844673:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949393.844674:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:3.0:1784949393.844678:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000020:00000040:0.0:1784949393.844681:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000020:00000001:0.0:1784949393.844715:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:1.0:1784949393.844720:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289694080:0@lo:400: 00000020:00000040:0.0:1784949393.844723:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000010:3.0:1784949393.844727:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 00000000d7cb9833. 00000020:00000001:0.0:1784949393.844732:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000020:00000001:0.0:1784949393.844743:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:1.0:1784949393.844744:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784949393.844752:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 00000100:00000040:0.0:1784949393.844760:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 02000000:00000001:1.0:1784949393.844761:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949393.844767:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1784949393.844770:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949393.844774:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=00000000f05bee52 refcount 3 to 1.2.3.4@tcp 00000400:00000200:3.0:1784949393.844781:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:0.0:1784949393.844783:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534560681152 : -115539148870464 : ffff96eaf212ecc0) 02000000:00000001:1.0:1784949393.844783:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1784949393.844795:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000010:1.0:1784949393.844798:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000004215ddb7. 00000400:00000200:3.0:1784949393.844801:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1784949393.844806:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 1.2.3.4@tcp) 02000000:00000001:1.0:1784949393.844810:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949393.844820:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 1.2.3.4@tcp/1.2.3.4@tcp 00000400:00000200:3.0:1784949393.844824:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000010:1.0:1784949393.844825:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 000000004667b736. 00000100:00000001:0.0:1784949393.844828:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949393.844830:0:337118:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1784949393.844838:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1784949393.844840:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1784949393.844840:0:337118:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d02c7fef. 00000400:00000010:1.0:1784949393.844846:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004aa0112e. 00000400:00000200:3.0:1784949393.844850:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:0.0:1784949393.844854:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949393.844858:0:337118:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1784949393.844860:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949393.844863:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289694080, portal 25 00000020:00000001:2.0:1784949393.844866:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:3.0:1784949393.844868:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3992:3992:234:4294967295] 1.2.3.4@tcp LPNI seq info [120:120:8:4294967295] 00000100:00000001:0.0:1784949393.844871:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000020:00000040:2.0:1784949393.844873:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000100:00000001:1.0:1784949393.844877:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949393.844879:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949393.844885:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:1.0:1784949393.844888:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469561856 : -115544239989760 : ffff96e9c29e9c00) 02000000:00000010:2.0:1784949393.844891:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000b1427c95. 00000100:00000010:0.0:1784949393.844893:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004614f9cb. 00000400:00000200:3.0:1784949393.844896:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000020:00000040:0.0:1784949393.844938:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 02000000:00000001:0.0:1784949393.844947:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1784949393.844949:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949393.844952:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949393.844956:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1784949393.844958:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949393.844962:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:2.0:1784949393.844963:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003dd0d971. 02000000:00000001:0.0:1784949393.844972:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949393.844976:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1784949393.844980:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949393.844982:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000100:00000001:2.0:1784949393.844983:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1784949393.844995:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001cc3fd72. 00000100:00000001:2.0:1784949393.844996:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.845011:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.845012:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949393.845020:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949393.845021:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:1.0:1784949393.845027:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1784949393.845030:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1784949393.845033:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:3.0:1784949393.845035:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000fc2fde8d. 00000100:00000001:2.0:1784949393.845037:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949393.845049:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000800:00000200:3.0:1784949393.845062:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000100:00000001:0.0:1784949393.845065:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1784949393.845065:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000100:00000001:0.0:1784949393.845072:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1784949393.845076:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 32 secs later) 00000100:00000200:1.0:1784949393.845080:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289694080, offset 0 00000100:00000040:0.0:1784949393.845085:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004614f9cb] to pc [ptlrpcd_rcv+-1] req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1784949393.845096:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000200:1.0:1784949393.845106:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000800:00000200:3.1:1784949393.845108:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1784949393.845129:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000800:00000010:3.0:1784949393.845133:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000fc2fde8d. 00000400:00000200:1.0:1784949393.845142:0:337121:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949393.845143:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949393.845146:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1784949393.845154:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1784949393.845164:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000100:00000001:0.0:1784949393.845167:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000200:1.0:1784949393.845174:0:337121:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c643ff80 00000100:00000010:0.0:1784949393.845177:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000aedb1960. 00000400:00000200:3.0:1784949393.845179:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000001:2.0:1784949393.845183:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1784949393.845189:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000001:2.0:1784949393.845192:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949393.845196:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 00000100:00000001:0.0:1784949393.845201:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949393.845201:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1784949393.845203:0:337121:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8ef9 [2] + 0 02000000:00000001:0.0:1784949393.845210:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1784949393.845211:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000001:0.0:1784949393.845217:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949393.845219:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949393.845229:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949393.845239:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949393.845255:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1784949393.845257:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000919ac10c. 00000100:00000001:2.0:1784949393.845260:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949393.845265:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949393.845275:0:337121:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 0000000037419771 02000000:00000001:2.0:1784949393.845276:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1784949393.845284:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949393.845285:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949393.845290:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949393.845292:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949393.845296:0:337121:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949393.845300:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:0.0:1784949393.845302:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aedb1960] to pc [ptlrpcd_00_00+0] req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949393.845307:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289694208:1.2.3.4@tcp:8: 00000400:00000200:3.0:1784949393.845318:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000400:00000200:1.0:1784949393.845322:0:337121:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:2.0:1784949393.845328:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949393.845337:0:337118:0:(niobuf.c:918:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949393.845338:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000200:1.0:1784949393.845344:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000037419771 00000100:00000001:2.0:1784949393.845346:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1784949393.845356:0:337121:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 0000000037419771. 00000100:00000001:2.0:1784949393.845358:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.845366:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1784949393.845372:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1784949393.845373:0:337121:0:(events.c:286:request_in_callback()) Process entered 00000100:00000001:2.0:1784949393.845374:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000001:3.0:1784949393.845378:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1784949393.845380:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.845382:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1784949393.845383:0:337121:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1784949393.845384:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000100:00000040:0.0:1784949393.845389:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11670) 00000400:00000010:3.0:1784949393.845395:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:2.0:1784949393.845396:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1784949393.845402:0:337121:0:(events.c:337:request_in_callback()) incoming req@00000000e3680464 x1871651289694080 msgsize 224 00000100:00000001:2.0:1784949393.845404:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1784949393.845407:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000040:0.0:1784949393.845408:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11670) 00000100:00000001:2.0:1784949393.845410:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.845420:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949393.845424:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 00000100:00100000:1.0:1784949393.845433:0:337121:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000010:3.0:1784949393.845437:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 00000100:00000040:1.0:1784949393.845441:0:337121:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949393.845448:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 00000100:00000001:0.0:1784949393.845453:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1784949393.845456:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000100:00100000:0.0:1784949393.845461:0:337120:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000400:00000200:3.0:1784949393.845466:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000100:00000001:1.0:1784949393.845467:0:337121:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949393.845468:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949393.845469:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949393.845480:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.845484:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949393.845489:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.845493:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949393.845495:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949393.845498:0:337120:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1784949393.845504:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000000cdd4381 00000100:00000040:0.0:1784949393.845508:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1784949393.845514:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000000cdd4381. 00000100:00000001:2.0:1784949393.845520:0:342755:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:2.0:1784949393.845529:0:342755:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289694080 00000100:00000001:1.0:1784949393.845530:0:337121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949393.845532:0:337120:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1784949393.845535:0:342755:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949393.845540:0:337120:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949393.845541:0:342755:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1784949393.845549:0:337120:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1784949393.845550:0:342755:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1784949393.845551:0:337121:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1784949393.845556:0:337120:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1784949393.845560:0:342755:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949393.845568:0:337120:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1784949393.845573:0:342755:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289694080 00000020:00000001:2.0:1784949393.845581:0:342755:0:(genops.c:813:class_conn2export()) Process entered 00000100:00100000:0.0:1784949393.845582:0:337120:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289694336:192.168.202.137@tcp:400: 00000020:00000040:2.0:1784949393.845588:0:342755:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:2.0:1784949393.845597:0:342755:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1784949393.845601:0:337120:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1784949393.845605:0:342755:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 02000000:00000001:0.0:1784949393.845609:0:337120:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1784949393.845617:0:337120:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949393.845617:0:342755:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:1.0:1784949393.845622:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1784949393.845629:0:337120:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1784949393.845630:0:342755:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000040:1.0:1784949393.845636:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1784949393.845638:0:337120:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000467d4d37. 00000100:00000001:2.0:1784949393.845641:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:0.0:1784949393.845649:0:337120:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949393.845653:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000100:00000001:2.0:1784949393.845664:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1784949393.845665:0:337120:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000005ecef72. 00000100:00000001:1.0:1784949393.845668:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949393.845680:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000100:00000001:1.0:1784949393.845681:0:337121:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:0.0:1784949393.845684:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000100:00000001:1.0:1784949393.845716:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1784949393.845721:0:342755:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c405a384. 00000100:00000001:1.0:1784949393.845725:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306333227 00000100:00000200:0.0:1784949393.845727:0:337120:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289694336, portal 10 00000020:00000010:2.0:1784949393.845732:0:342755:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000b49ef65d. 00000100:00000010:1.0:1784949393.845738:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000291e02ca. 00000100:00000001:0.0:1784949393.845741:0:337120:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000010:2.0:1784949393.845746:0:342755:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000521c0956. 00000100:00000001:0.0:1784949393.845749:0:337120:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469559168 : -115544239992448 : ffff96e9c29e9180) 00000100:00000001:1.0:1784949393.845753:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000040:2.0:1784949393.845760:0:342755:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:2.0:1784949393.845768:0:342755:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:1.0:1784949393.845770:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306333227 00000100:00000040:0.0:1784949393.845777:0:337120:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949393.845777:0:342755:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000010:1.0:1784949393.845781:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000006a83a85b. 00000100:00000001:2.0:1784949393.845784:0:342755:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949393.845792:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1784949393.845802:0:342755:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949393.845803:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306333227 00000100:00000010:1.0:1784949393.845817:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000070432f3d. 00000100:00000001:0.0:1784949393.845818:0:337120:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1784949393.845827:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1784949393.845828:0:342755:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949393.845837:0:337120:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:1.0:1784949393.845840:0:337121:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306333227 00000400:00000010:0.0:1784949393.845850:0:337120:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000045d823da. 00000100:00000010:1.0:1784949393.845851:0:337121:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000de58da65. 00000100:00000001:2.0:1784949393.845852:0:342755:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:2.0:1784949393.845862:0:342755:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:0.0:1784949393.845864:0:337120:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289694336, offset 0 00000100:00000001:1.0:1784949393.845865:0:337121:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000100:00100000:2.0:1784949393.845877:0:342755:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 843 00000100:00000001:1.0:1784949393.845880:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1784949393.845883:0:342755:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000100:00000001:1.0:1784949393.845889:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949393.845891:0:337120:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949393.845893:0:342755:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782162000 : -115538927389616 : ffff96eaff467450) 00000100:00000001:1.0:1784949393.845899:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949393.845948:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949393.845961:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000e3680464 x1871651289694080/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:299/0 lens 224/0 e 0 to 0 dl 1784949404 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949393.845966:0:337120:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949393.845983:0:337120:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784949393.845988:0:342755:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000400:00000200:0.0:1784949393.845991:0:337120:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:2.0:1784949393.845997:0:342755:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000f3d47e6e. 00000400:00000200:0.0:1784949393.845998:0:337120:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1784949393.846008:0:342755:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1784949393.846010:0:337120:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000020:00000010:2.0:1784949393.846017:0:342755:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000f3d47e6e. 00000400:00000200:0.0:1784949393.846021:0:337120:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3993:3993:234:4294967295] 192.168.202.137@tcp LPNI seq info [2542:2542:8:4294967295] 00000100:00100000:2.0:1784949393.846032:0:342755:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000e3680464 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289694080:12345-0@lo:400: 00000100:00000200:2.0:1784949393.846047:0:342755:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289694080 00000400:00000200:0.0:1784949393.846049:0:337120:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000020:00000001:2.0:1784949393.846053:0:342755:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000800:00000200:0.0:1784949393.846061:0:337120:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000020:00000001:2.0:1784949393.846062:0:342755:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000800:00000010:0.0:1784949393.846069:0:337120:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000005a1c05a3. 00000020:00000001:2.0:1784949393.846070:0:342755:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949393.846080:0:342755:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000800:00000200:0.0:1784949393.846081:0:337120:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000020:00000001:2.0:1784949393.846088:0:342755:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000800:00000200:0.0:1784949393.846098:0:337120:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000020:00000001:2.0:1784949393.846100:0:342755:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000800:00000200:0.0:1784949393.846105:0:337120:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000005a1c05a3 type 1, nob 320 niov 1 nkiov 1 00000020:00000001:2.0:1784949393.846108:0:342755:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1784949393.846118:0:342755:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:2.0:1784949393.846122:0:342755:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784949393.846128:0:337120:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949393.846128:0:342755:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949393.846143:0:337120:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2265 usecs (3 jiffies) 00000020:00000001:2.0:1784949393.846143:0:342755:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949393.846153:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306333189 00000100:00000001:2.0:1784949393.846154:0:342755:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949393.846158:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306333227 00000100:00000001:0.0:1784949393.846164:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306333189 02000000:00000001:2.0:1784949393.846164:0:342755:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949393.846173:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306333227 00000100:00000001:0.0:1784949393.846177:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306333189 02000000:00000010:2.0:1784949393.846178:0:342755:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000077ad1613. 00000100:00000001:0.0:1784949393.846180:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306333227 00000100:00000001:0.0:1784949393.846190:0:337120:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306333189 02000000:00000001:2.0:1784949393.846192:0:342755:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.846194:0:337120:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306333227 00000100:00000001:0.0:1784949393.846201:0:337120:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949393.846204:0:342755:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.846206:0:337120:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1784949393.846217:0:342755:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.846218:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1784949393.846231:0:342755:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289694080 00000100:00000001:0.0:1784949393.846232:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949393.846244:0:342755:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:2.0:1784949393.846255:0:342755:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000e3680464 x1871651289694080/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:299/0 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1784949393.846275:0:342755:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:2.0:1784949393.846279:0:342755:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949393.846285:0:342755:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=115 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949393.846293:0:342755:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1784949393.846300:0:342755:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949393.846305:0:342755:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000200:3.0:1784949393.846309:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000001:2.0:1784949393.846312:0:342755:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000800:00000010:3.0:1784949393.846317:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000005a1c05a3. 02000000:00000001:2.0:1784949393.846319:0:342755:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949393.846328:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1784949393.846329:0:342755:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1784949393.846343:0:342755:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000200:3.0:1784949393.846343:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000010:2.0:1784949393.846356:0:342755:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000200:3.0:1784949393.846357:0:337112:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000045d823da 00000400:00000010:3.0:1784949393.846363:0:337112:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000045d823da. 00000100:00000001:3.0:1784949393.846370:0:337112:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1784949393.846376:0:337112:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1784949393.846383:0:342755:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289694080, offset 224 00000100:00000001:3.0:1784949393.846401:0:337112:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000001:1.0:1784949393.846407:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949393.846411:0:342755:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000100:00000040:3.0:1784949393.846411:0:337112:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949393.846428:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949393.846429:0:337112:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1784949393.846442:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:2.0:1784949393.846443:0:342755:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:3.0:1784949393.846443:0:337112:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1784949393.846451:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1784949393.846472:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949393.846476:0:342755:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c643ff80 00000400:00000200:1.0:1784949393.846500:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6440080 00000400:00000200:2.0:1784949393.846501:0:342755:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9911 [1] + 224 00000400:00000200:2.0:1784949393.846521:0:342755:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949393.846531:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 184496 00000400:00000200:2.0:1784949393.846533:0:342755:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000800:00000001:1.0:1784949393.846551:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949393.846552:0:342755:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:1.0:1784949393.846564:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1784949393.846565:0:342755:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1784949393.846572:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949393.846589:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:1.0:1784949393.846605:0:337114:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:1.0:1784949393.846611:0:337114:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000040:2.0:1784949393.846617:0:342755:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1784949393.846620:0:337114:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000080e08afd. 00000100:00000040:1.0:1784949393.846628:0:337114:0:(events.c:337:request_in_callback()) incoming req@0000000080e08afd x1871651289694336 msgsize 224 00000100:00100000:1.0:1784949393.846650:0:337114:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 00000100:00000001:1.0:1784949393.846683:0:337114:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1784949393.846707:0:342755:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949393.846722:0:342755:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000800:00000001:1.0:1784949393.846737:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784949393.846740:0:342755:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000800:00000001:1.0:1784949393.846746:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949393.846751:0:342755:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949393.846758:0:342755:0:(events.c:398:reply_out_callback()) Process leaving 00000100:00000001:2.0:1784949393.846768:0:342755:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949393.846774:0:342764:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000040:2.0:1784949393.846778:0:342755:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000100:00100000:3.0:1784949393.846783:0:342764:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289694336 00010000:00000001:2.0:1784949393.846787:0:342755:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 02000000:00000001:3.0:1784949393.846792:0:342764:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000020:00000001:2.0:1784949393.846794:0:342755:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.846797:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1784949393.846798:0:342764:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000020:00000001:2.0:1784949393.846806:0:342755:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000001:0.0:1784949393.846807:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1784949393.846808:0:342764:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1784949393.846819:0:342764:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1784949393.846822:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1784949393.846825:0:342755:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e3680464 x1871651289694080/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:299/0 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949393.846827:0:342764:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289694336 00000400:00000001:0.0:1784949393.846828:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949393.846836:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 000000004667b736. 00000020:00000001:3.0:1784949393.846837:0:342764:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1784949393.846843:0:342764:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000400:00000200:0.0:1784949393.846848:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004aa0112e 00000020:00000001:3.0:1784949393.846850:0:342764:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1784949393.846857:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000004aa0112e. 00000020:00000040:3.0:1784949393.846860:0:342764:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:0.0:1784949393.846868:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1784949393.846871:0:342764:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000020:00000001:3.0:1784949393.846878:0:342764:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 00000100:00000200:0.0:1784949393.846882:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949393.846889:0:342755:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000e3680464 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337121:x1871651289694080:12345-0@lo:400: Request processed in 890us (1475us total) trans 0 rc 0/0 00000100:00000001:3.0:1784949393.846891:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949393.846901:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dd6c0379. 00000100:00000001:3.0:1784949393.846943:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1784949393.846953:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dd6c0379. 00000020:00000010:3.0:1784949393.846970:0:342764:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000089ec28da. 00000100:00000200:0.0:1784949393.846973:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949393.846975:0:342755:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 843 00000020:00000010:3.0:1784949393.846981:0:342764:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000a4e0290. 00000100:00000040:2.0:1784949393.846991:0:342755:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000020:00000010:3.0:1784949393.846993:0:342764:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000100:00000001:0.0:1784949393.847002:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949393.847005:0:342755:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949393.847008:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949393.847015:0:342755:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000040:3.0:1784949393.847015:0:342764:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949393.847016:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949393.847027:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1784949393.847027:0:342764:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000020:00000010:2.0:1784949393.847030:0:342755:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000b49ef65d. 00000100:00000001:3.0:1784949393.847033:0:342764:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949393.847035:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949393.847042:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949393.847043:0:342764:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1784949393.847046:0:342755:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000521c0956. 02000000:00000001:0.0:1784949393.847047:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949393.847053:0:342764:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00001000:0.0:1784949393.847057:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=115 v=5 (5 5 5 5) 00000100:00000001:3.0:1784949393.847059:0:342764:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000020:00000010:2.0:1784949393.847061:0:342755:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c405a384. 00000100:00000001:3.0:1784949393.847069:0:342764:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949393.847070:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=115 v=5 (1 1 1 1) 00000020:00000040:2.0:1784949393.847077:0:342755:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000100:00000001:0.0:1784949393.847083:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949393.847087:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949393.847090:0:342755:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00010000:00000001:0.0:1784949393.847092:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00000100:00000001:3.0:1784949393.847096:0:342764:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1784949393.847099:0:337121:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1784949393.847100:0:342755:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000077ad1613. 00000100:00000001:0.0:1784949393.847109:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949393.847113:0:342755:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000001:3.0:1784949393.847122:0:342764:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000040:0.0:1784949393.847123:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949393.847123:0:342755:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1784949393.847129:0:342764:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000200:2.0:1784949393.847141:0:342755:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000001:0.0:1784949393.847144:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949393.847151:0:342762:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00100000:3.0:1784949393.847152:0:342764:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 819 00000100:00000001:0.0:1784949393.847155:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1784949393.847159:0:342762:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000400:00000010:2.0:1784949393.847160:0:342755:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000080bb5714. 00000100:00000001:0.0:1784949393.847161:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949393.847166:0:342764:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:1.0:1784949393.847167:0:342762:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949393.847174:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1784949393.847178:0:342755:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fe8db63d. 00000100:00000001:1.0:1784949393.847179:0:342762:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949393.847179:0:342764:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554890624 : -115544154660992 : ffff96e9c7b49f80) 00000100:00000040:3.0:1784949393.847203:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@0000000080e08afd x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:299/0 lens 224/0 e 0 to 0 dl 1784949404 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949393.847204:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337121:1871651289694080:0@lo:400: 00000100:00000001:0.0:1784949393.847223:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.847232:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949393.847241:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000b83ea808 x1871651289694080/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949393.847242:0:342764:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949393.847249:0:342764:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dd6c0379. 00000100:00000001:3.0:1784949393.847257:0:342764:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949393.847265:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000010:3.0:1784949393.847266:0:342764:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dd6c0379. 02000000:00000001:0.0:1784949393.847272:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949393.847278:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000004215ddb7. 00000100:00100000:3.0:1784949393.847280:0:342764:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@0000000080e08afd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289694336:12345-192.168.202.137@tcp:400: 02000000:00000001:0.0:1784949393.847287:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:3.0:1784949393.847289:0:342764:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289694336 00000020:00000001:0.0:1784949393.847292:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:3.0:1784949393.847295:0:342764:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000040:0.0:1784949393.847297:0:337121:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:0.0:1784949393.847304:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:3.0:1784949393.847305:0:342764:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 02000000:00000010:0.0:1784949393.847308:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002a753d9d. 00000020:00000001:3.0:1784949393.847309:0:342764:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949393.847320:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1784949393.847320:0:342764:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 02000000:00000001:0.0:1784949393.847326:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1784949393.847328:0:342764:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000100:00000010:0.0:1784949393.847332:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000b83ea808. 00000020:00000001:3.0:1784949393.847335:0:342764:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000001:0.0:1784949393.847340:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1784949393.847341:0:342764:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.847346:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949393.847348:0:342764:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:0.0:1784949393.847354:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1784949393.847355:0:342764:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:3.0:1784949393.847362:0:342764:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.847365:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1784949393.847366:0:342764:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949393.847371:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949393.847375:0:342764:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000100:00000001:0.0:1784949393.847377:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1784949393.847380:0:342764:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949393.847382:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1784949393.847388:0:342764:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000017ac11c7. 02000000:00000001:3.0:1784949393.847393:0:342764:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949393.847396:0:342764:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949393.847399:0:342764:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1784949393.847402:0:342764:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289694336 00010000:00000001:3.0:1784949393.847406:0:342764:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:3.0:1784949393.847411:0:342764:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@0000000080e08afd x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:299/0 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1784949393.847421:0:342764:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949393.847424:0:342764:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949393.847427:0:342764:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=110 v=5 (1 1 1 1) 00000100:00000001:3.0:1784949393.847431:0:342764:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1784949393.847434:0:342764:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:3.0:1784949393.847438:0:342764:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:3.0:1784949393.847441:0:342764:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:3.0:1784949393.847443:0:342764:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949393.847446:0:342764:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949393.847449:0:342764:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:3.0:1784949393.847452:0:342764:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000003d1188f5. 00000100:00000200:3.0:1784949393.847456:0:342764:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289694336, offset 224 00000400:00000200:3.0:1784949393.847461:0:342764:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:3.0:1784949393.847470:0:342764:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949393.847477:0:342764:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3994:3994:234:4294967295] 192.168.202.137@tcp LPNI seq info [2543:2543:8:4294967295] 00000400:00000200:3.0:1784949393.847488:0:342764:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:3.0:1784949393.847493:0:342764:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949393.847497:0:342764:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000fc2fde8d. 00000800:00000200:3.0:1784949393.847502:0:342764:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949393.847509:0:342764:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949393.847512:0:342764:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000fc2fde8d type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1784949393.847530:0:342764:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949393.847534:0:342764:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 6 to 192.168.202.137@tcp 00010000:00000001:3.0:1784949393.847537:0:342764:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:3.0:1784949393.847539:0:342764:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949393.847541:0:342764:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:3.0:1784949393.847545:0:342764:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000080e08afd x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:299/0 lens 224/224 e 0 to 0 dl 1784949404 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949393.847558:0:342764:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@0000000080e08afd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:337120:x1871651289694336:12345-192.168.202.137@tcp:400: Request processed in 316us (926us total) trans 0 rc 0/0 00000100:00100000:3.0:1784949393.847569:0:342764:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 819 00000100:00000040:3.0:1784949393.847574:0:342764:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:3.0:1784949393.847577:0:342764:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1784949393.847579:0:342764:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:3.0:1784949393.847583:0:342764:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000a4e0290. 00000020:00000010:3.0:1784949393.847588:0:342764:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000020:00000010:3.0:1784949393.847592:0:342764:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000089ec28da. 00000020:00000040:3.0:1784949393.847598:0:342764:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:3.0:1784949393.847602:0:342764:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1784949393.847713:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1784949393.847720:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000fc2fde8d. 00000400:00000200:0.0:1784949393.847726:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1784949393.847735:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000800:00000001:3.0:1784949393.847738:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000200:0.0:1784949393.847741:0:337113:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003d1188f5 00000400:00000010:0.0:1784949393.847747:0:337113:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 000000003d1188f5. 00000100:00000001:0.0:1784949393.847751:0:337113:0:(events.c:386:reply_out_callback()) Process entered 00000800:00000001:3.0:1784949393.847753:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.847755:0:337113:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:3.0:1784949393.847761:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949393.847769:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949393.847778:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:3.0:1784949393.847791:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6440080 00000400:00000200:3.0:1784949393.847798:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9919 [1] + 224 00000800:00000001:3.0:1784949393.847805:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949393.847816:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949393.847819:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949393.847823:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949393.847828:0:337112:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949393.847833:0:337112:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949393.847844:0:337112:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949393.847866:0:337112:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1784949393.847873:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949393.847876:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.847935:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949393.847939:0:337120:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949393.847944:0:337120:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949393.847946:0:337120:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949393.847948:0:337120:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000005ecef72. 00000400:00000200:0.0:1784949393.847953:0:337120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000400:00000010:0.0:1784949393.847955:0:337120:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:0.0:1784949393.847958:0:337120:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949393.847964:0:337120:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949393.847975:0:337120:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949393.847984:0:337120:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949393.847986:0:337120:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.847988:0:337120:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949393.847990:0:337120:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949393.847992:0:337120:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949393.847993:0:337120:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949393.847996:0:337120:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949393.848002:0:337120:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=110 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949393.848005:0:337120:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=110 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949393.848008:0:337120:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949393.848010:0:337120:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949393.848012:0:337120:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949393.848016:0:337120:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949393.848025:0:337120:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.848028:0:337120:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949393.848030:0:337120:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949393.848033:0:337120:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949393.848034:0:337120:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949393.848038:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949393.848046:0:337120:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949393.848049:0:337120:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949393.848050:0:337120:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949393.848053:0:337120:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949393.848061:0:337120:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:337120:1871651289694336:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949393.848066:0:337120:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.848069:0:337120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949393.848071:0:337120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000aedb1960 x1871651289694336/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949409 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949393.848078:0:337120:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949393.848080:0:337120:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949393.848082:0:337120:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000467d4d37. 02000000:00000001:0.0:1784949393.848085:0:337120:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949393.848086:0:337120:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949393.848088:0:337120:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949393.848091:0:337120:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949393.848092:0:337120:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:0.0:1784949393.848095:0:337120:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949393.848097:0:337120:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949393.848098:0:337120:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000aedb1960. 00000100:00000001:0.0:1784949393.848101:0:337120:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949393.848103:0:337120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.848105:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949393.848109:0:337120:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949393.848111:0:337120:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949393.848113:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949393.848115:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949393.887959:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306333227, at 4306333228 00000100:00000040:3.0:1784949393.887962:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306333227, at 4306333228 00000100:00000010:1.0:1784949393.887969:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000006a83a85b. 00000100:00000010:3.0:1784949393.887973:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000de58da65. 00000100:00000040:2.0:1784949393.887975:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306333227, at 4306333228 00000100:00000040:0.0:1784949393.887994:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306333227, at 4306333228 00000100:00000010:2.0:1784949393.888000:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000070432f3d. 00000100:00000010:0.0:1784949393.888005:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000291e02ca. 00010000:00000001:2.0:1784949394.676012:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949394.676021:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949394.676028:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949394.676030:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949394.676178:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949394.676185:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949394.676189:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1784949394.868058:0:337118:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1784949394.868069:0:337118:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1784949394.868086:0:337118:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1784949393/real 0] req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949394 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949394.868106:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.868109:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949394.868112:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1784949394.868133:0:337118:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949394 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949394.868143:0:337118:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.868153:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949394.868155:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.868158:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949394.868161:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1784949394.868163:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949394.868167:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949394 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949394.868175:0:337118:0:(client.c:2129:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1784949394.868179:0:337118:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.868180:0:337118:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949394.868182:0:337118:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.868184:0:337118:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1784949394.868186:0:337118:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1784949394.868190:0:337118:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1784949394.868220:0:337118:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 1.2.3.4@tcp failed (-110) 00000100:00000001:2.0:1784949394.868224:0:337118:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1784949394.868229:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949394 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1784949394.868238:0:337118:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000004614f9cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289694208:1.2.3.4@tcp:8: 00000100:00000001:2.0:1784949394.868245:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.868248:0:337118:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1784949394.868252:0:337118:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000004614f9cb x1871651289694208/t0(0) o8->lustre-OST0000-osc-MDT0000@1.2.3.4@tcp:28/4 lens 520/544 e 0 to 1 dl 1784949394 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949394.868261:0:337118:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1784949394.868265:0:337118:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1784949394.868268:0:337118:0:(genops.c:1244:class_import_put()) import 000000002df1e6db refcount=2 obd=lustre-OST0000-osc-MDT0000 00000020:00000001:2.0:1784949394.868271:0:337118:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:0.0:1784949394.868273:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:2.0:1784949394.868276:0:337118:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 000000001cc3fd72. 00000100:00000001:0.0:1784949394.868295:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:2.0:1784949394.868298:0:337118:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949394.868302:0:337118:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949394.868304:0:337118:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004614f9cb. 00000100:00000010:0.0:1784949394.868308:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000aedb1960. 00000100:00000001:2.0:1784949394.868311:0:337118:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000040:0.0:1784949394.868317:0:335217:0:(genops.c:1231:class_import_get()) import 0000000036c7f24e refcount=4 obd=MGC192.168.202.137@tcp 00000100:00000001:2.0:1784949394.868318:0:337118:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.868326:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.868331:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1784949394.868333:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949394.868337:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949394.868339:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1784949394.868342:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949394.868346:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949394.868347:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1784949394.868359:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 0000000036e798e7. 00000100:00000001:0.0:1784949394.868368:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949394.868378:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1d45fc09-7805-4bc2-ab31-cf64613dd99c->MGS req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1784949394.868391:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1784949394.868394:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949394.868400:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aedb1960] to pc [ptlrpcd_00_01+1] req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949394.868438:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949394.868442:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 0 suppress 1 00000100:00080000:0.0:1784949394.868448:0:335217:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1784949394.868452:0:335217:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1784949394.868456:0:335217:0:(import.c:65:import_set_state_nolock()) 000000002df1e6db lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1784949394.868487:0:335217:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1784949394.868495:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 192.168.202.137@tcp last attempt 11640 00000100:00000001:2.0:1784949394.868501:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1784949394.868507:0:337119:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000400:00000200:0.0:1784949394.868511:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 192.168.202.137@tcp is discovered: rc = 1 00000100:00000001:2.0:1784949394.868513:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1784949394.868521:0:335217:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 1.2.3.4@tcp last attempt 11665 00000100:00000001:1.0:1784949394.868531:0:337120:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949394.868532:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949394.868535:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.868541:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949394.868542:0:337120:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.868545:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1784949394.868546:0:335217:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: 0 00000400:00000200:0.0:1784949394.868556:0:335217:0:(peer.c:1600:LNetPeerDiscovered()) Peer NID 1.2.3.4@tcp is not discovered: rc = -11 00000100:00000001:2.0:1784949394.868563:0:337119:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1784949394.868567:0:335217:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1784949394.868573:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1784949394.868579:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 2 to 1.2.3.4@tcp 00000100:00000001:0.0:1784949394.868586:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949394.868610:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 7 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949394.868619:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000020:00000001:0.0:1784949394.868630:0:335217:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949394.868636:0:335217:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad5cf 00000100:00000001:2.0:1784949394.868636:0:337119:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949394.868643:0:337119:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949394.868644:0:335217:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1784949394.868653:0:337119:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1784949394.868654:0:335217:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000c1b5b592 refcount=4 00000400:00000200:3.0:1784949394.868655:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 02000000:00000001:2.0:1784949394.868660:0:337119:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1784949394.868667:0:335217:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 02000000:00000001:2.0:1784949394.868669:0:337119:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949394.868675:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 00000100:00100000:2.0:1784949394.868696:0:337119:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289694464:0@lo:400: 00000020:00000001:0.0:1784949394.868699:0:335217:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534561374208 : -115539148177408 : ffff96eaf21d8000) 00000100:00000001:2.0:1784949394.868706:0:337119:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1784949394.868714:0:335217:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=00000000f05bee52 refcount 1 to 1.2.3.4@tcp 02000000:00000001:2.0:1784949394.868717:0:337119:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1784949394.868721:0:335217:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1784949394.868723:0:337119:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1784949394.868725:0:337117:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 0000000089ec28da. 00000100:00000040:0.0:1784949394.868728:0:335217:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 02000000:00000001:2.0:1784949394.868731:0:337119:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1784949394.868739:0:335217:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 00000400:00000010:3.0:1784949394.868743:0:337117:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000020:00000040:0.0:1784949394.868744:0:335217:0:(genops.c:900:class_export_put()) PUTting export 00000000c1b5b592 : new refcount 3 02000000:00000010:2.0:1784949394.868745:0:337119:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000c405a384. 02000000:00000001:2.0:1784949394.868756:0:337119:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1784949394.868757:0:335217:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: Connection changing to lustre-OST0000 (at 192.168.202.137@tcp) 00000100:00080000:0.0:1784949394.868763:0:335217:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-MDT0000: import 000000002df1e6db using connection 192.168.202.137@tcp/192.168.202.137@tcp 00000100:00000001:0.0:1784949394.868767:0:335217:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949394.868768:0:337119:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000603fb577. 00000400:00000010:3.0:1784949394.868769:0:337117:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008ae4e4cb. 02000000:00000001:0.0:1784949394.868774:0:335217:0:(sec.c:1589:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1784949394.868786:0:335217:0:(sec.c:1629:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1784949394.868786:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000200:3.0:1784949394.868787:0:337117:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-1.2.3.4@tcp 02000000:00000001:0.0:1784949394.868798:0:335217:0:(sec.c:1657:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.868802:0:335217:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:2.0:1784949394.868805:0:337119:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289694464, portal 25 00000100:00000001:0.0:1784949394.868806:0:335217:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.868814:0:337119:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1784949394.868816:0:337117:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 1.2.3.4@tcp local destination 00000100:00000001:2.0:1784949394.868819:0:337119:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469559168 : -115544239992448 : ffff96e9c29e9180) 00000100:00000010:0.0:1784949394.868820:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b83ea808. 00000400:00000200:3.0:1784949394.868826:0:337117:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:0.0:1784949394.868830:0:335217:0:(genops.c:1231:class_import_get()) import 000000002df1e6db refcount=3 obd=lustre-OST0000-osc-MDT0000 00000100:00000040:2.0:1784949394.868838:0:337119:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949394.868839:0:337117:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 02000000:00000001:0.0:1784949394.868841:0:335217:0:(sec.c:2067:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1784949394.868848:0:335217:0:(sec.c:2077:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949394.868853:0:337117:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1784949394.868860:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1784949394.868866:0:337117:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 1.2.3.4@tcp 02000000:00000001:0.0:1784949394.868868:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1784949394.868875:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.868876:0:337119:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1784949394.868884:0:337117:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3995:3995:234:4294967295] 1.2.3.4@tcp LPNI seq info [121:121:8:4294967295] 02000000:00000010:0.0:1784949394.868888:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000151697a2. 00000100:00000040:2.0:1784949394.868889:0:337119:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:2.0:1784949394.868899:0:337119:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000985b7fdb. 00000100:00000200:2.0:1784949394.868954:0:337119:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1871651289694464, offset 0 00000100:00000001:0.0:1784949394.868958:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949394.868965:0:337119:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949394.868971:0:337117:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 1.2.3.4@tcp 00000100:00100000:0.0:1784949394.868977:0:335217:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b83ea808 x1871651289694592/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1784949394.868989:0:337117:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 1.2.3.4@tcp(1.2.3.4@tcp:1.2.3.4@tcp) : GET try# 0 00000800:00000200:3.0:1784949394.869009:0:337117:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-1.2.3.4@tcp 00000100:00000001:0.0:1784949394.869012:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1784949394.869018:0:337119:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000100:00000001:0.0:1784949394.869019:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1784949394.869021:0:337117:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000c9dda688. 00000100:00000040:0.0:1784949394.869037:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b83ea808] to pc [ptlrpcd_rcv+-1] req@00000000b83ea808 x1871651289694592/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1784949394.869039:0:337119:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x6a641c6440100 00000800:00000200:3.0:1784949394.869042:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000800:00000200:3.0:1784949394.869052:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 31 secs later) 00000400:00000200:2.0:1784949394.869059:0:337119:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0x8f15 [2] + 0 00000800:00000200:3.1:1784949394.869072:0:337117:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000067bb1f2] -> 12345-1.2.3.4@tcp (2) 00000400:00000010:2.0:1784949394.869073:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000037f9e020. 00000800:00000200:3.1:1784949394.869084:0:337117:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 1.2.3.4 (cnted 0, interval 60, 31 secs later) 00000400:00000200:2.0:1784949394.869086:0:337119:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000007e436012 00000800:00000100:3.0:1784949394.869095:0:337117:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-1.2.3.4@tcp 00000400:00000200:2.0:1784949394.869099:0:337119:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000010:3.0:1784949394.869100:0:337117:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000c9dda688. 00000100:00000001:0.0:1784949394.869107:0:335217:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949394.869111:0:337117:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1784949394.869116:0:337119:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000040:0.0:1784949394.869117:0:335217:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1784949394.869127:0:337117:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->1.2.3.4@tcp: GET: REMOTE_ERROR 00000400:00000200:2.0:1784949394.869128:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007e436012 00000100:00000001:0.0:1784949394.869129:0:335217:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000400:00000010:2.0:1784949394.869136:0:337119:0:(lib-lnet.h:280:lnet_md_free()) kfreed 'md': 152 at 000000007e436012. 00000100:00000001:1.0:1784949394.869137:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1784949394.869138:0:335217:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000003bf8e8ad. 00000100:00000001:2.0:1784949394.869145:0:337119:0:(events.c:286:request_in_callback()) Process entered 00000400:00000200:3.0:1784949394.869145:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000100:00000001:1.0:1784949394.869146:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1784949394.869147:0:335217:0:(genops.c:1231:class_import_get()) import 000000006c89a515 refcount=3 obd=lustre-MDT0000-lwp-MDT0000 00000100:00000200:2.0:1784949394.869154:0:337119:0:(events.c:296:request_in_callback()) event type 2, status 0, service mgs 00000100:00000001:1.0:1784949394.869155:0:337118:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1784949394.869156:0:335217:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1784949394.869159:0:337117:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 1.2.3.4@tcp: 1 02000000:00000001:0.0:1784949394.869164:0:335217:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1784949394.869167:0:337119:0:(events.c:337:request_in_callback()) incoming req@000000009bc35acb x1871651289694464 msgsize 224 02000000:00000001:0.0:1784949394.869171:0:335217:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1784949394.869171:0:337118:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000b83ea808 x1871651289694592/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949394.869172:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000010:0.0:1784949394.869184:0:335217:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at 000000002a753d9d. 00000100:00100000:2.0:1784949394.869188:0:337119:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00000001:0.0:1784949394.869194:0:335217:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949394.869196:0:337117:0:(peer.c:3937:lnet_peer_send_ping()) peer 1.2.3.4@tcp 00000100:00000040:2.0:1784949394.869198:0:337119:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000400:00000200:3.0:1784949394.869208:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 rc 0 00000100:00000040:0.0:1784949394.869212:0:335217:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging lustre-MDT0000-lwp-MDT0000_UUID->lustre-MDT0000_UUID req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1784949394.869217:0:337118:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1784949394.869218:0:337119:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:1.0:1784949394.869225:0:337118:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949394.869225:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x100861 00000400:00000200:2.0:1784949394.869231:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000985b7fdb 02000000:00000001:1.0:1784949394.869235:0:337118:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:2.0:1784949394.869238:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000985b7fdb. 02000000:00000001:1.0:1784949394.869242:0:337118:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949394.869248:0:337119:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1784949394.869249:0:335217:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1784949394.869250:0:337118:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784949394.869250:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1784949394.869256:0:335217:0:(jobid.c:947:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1784949394.869257:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:2.0:1784949394.869258:0:337119:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949394.869263:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000100:00100000:1.0:1784949394.869264:0:337118:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@00000000b83ea808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:337118:1871651289694592:192.168.202.137@tcp:8: 00000400:00000010:3.0:1784949394.869270:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000100:00000040:0.0:1784949394.869272:0:335217:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bf8e8ad] to pc [ptlrpcd_00_02+2] req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1784949394.869287:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000100:00000001:1.0:1784949394.869292:0:337118:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1784949394.869300:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 1.2.3.4@tcp: -114 02000000:00000001:1.0:1784949394.869301:0:337118:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1784949394.869308:0:337118:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.869313:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1784949394.869313:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000089ec28da. 02000000:00000001:1.0:1784949394.869316:0:337118:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1784949394.869323:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1784949394.869327:0:337118:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000000b08bf97. 00000400:00000200:3.0:1784949394.869329:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 1.2.3.4@tcp:-5 02000000:00000001:1.0:1784949394.869336:0:337118:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.869340:0:335217:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1784949394.869343:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 1.2.3.4@tcp(000000006fa298b4) state 0x102061 rc -5 00000400:00000010:1.0:1784949394.869348:0:337118:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000789ee428. 00000100:00000040:0.0:1784949394.869351:0:335217:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11671) 00000400:00000200:3.0:1784949394.869358:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 1.2.3.4@tcp 00000400:00000010:1.0:1784949394.869361:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000000cdd4381. 00000100:00000001:2.0:1784949394.869362:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.869369:0:337119:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:1.0:1784949394.869384:0:337118:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1871651289694592, portal 4 00000100:00000001:3.0:1784949394.869385:0:337122:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949394.869386:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949394.869386:0:337119:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949394.869390:0:337118:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1784949394.869395:0:337122:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.869396:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1784949394.869396:0:337118:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469561856 : -115544239989760 : ffff96e9c29e9c00) 00000100:00000001:2.0:1784949394.869397:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 0 is 4306334251 00000100:00000001:0.0:1784949394.869402:0:337121:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:2.0:1784949394.869411:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000882090e1. 00000100:00000040:0.0:1784949394.869413:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1784949394.869417:0:337118:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@00000000b83ea808 x1871651289694592/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949419 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949394.869426:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1784949394.869440:0:342756:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00000001:2.0:1784949394.869441:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 1 is 4306334251 00000100:00100000:3.0:1784949394.869447:0:342756:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289694464 00000100:00000010:2.0:1784949394.869452:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b9b6859c. 02000000:00000001:3.0:1784949394.869455:0:342756:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00000001:0.0:1784949394.869463:0:337121:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1784949394.869463:0:342756:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949394.869464:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1784949394.869472:0:337121:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949394.869479:0:342756:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.869481:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 2 is 4306334251 02000000:00000001:0.0:1784949394.869489:0:337121:0:(sec.c:719:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1784949394.869490:0:337118:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:2.0:1784949394.869491:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f99e6ef2. 02000000:00000001:3.0:1784949394.869493:0:342756:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949394.869498:0:337121:0:(sec.c:750:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1784949394.869501:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 2 00000100:00100000:3.0:1784949394.869504:0:342756:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289694464 00000100:00000040:1.0:1784949394.869505:0:337118:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949394.869509:0:337119:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS new deadline estimation for cpu 3 is 4306334251 02000000:00000001:0.0:1784949394.869512:0:337121:0:(sec.c:794:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949394.869513:0:342756:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:1.0:1784949394.869516:0:337118:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000a0c4f329. 00000100:00000010:2.0:1784949394.869517:0:337119:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f59314ac. 00000020:00000040:3.0:1784949394.869518:0:342756:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad54a 00000020:00000001:3.0:1784949394.869526:0:342756:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:0.0:1784949394.869527:0:337121:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289694720:192.168.202.137@tcp:400: 00000100:00000200:1.0:1784949394.869527:0:337118:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1871651289694592, offset 0 00000100:00000001:2.0:1784949394.869528:0:337119:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.137@tcp: PM QoS request now active for cpu 3 00000020:00000040:3.0:1784949394.869534:0:342756:0:(lustre_handles.c:147:class_handle2object()) GET export 00000000b1518b45 refcount=12 00000100:00000001:2.0:1784949394.869539:0:337119:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1784949394.869544:0:342756:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000400:00000200:1.0:1784949394.869548:0:337118:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:2.0:1784949394.869551:0:337119:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949394.869554:0:342756:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628534733328384 : -115538976223232 : ffff96eafc5d5000) 00000100:00000001:3.0:1784949394.869568:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000100:00000001:0.0:1784949394.869590:0:337121:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1784949394.869591:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949394.869595:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dd6c0379. 02000000:00000001:0.0:1784949394.869598:0:337121:0:(sec.c:1090:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1784949394.869600:0:337118:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784949394.869601:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949394.869605:0:337121:0:(sec.c:1127:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949394.869605:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1784949394.869618:0:337121:0:(sec.c:1874:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:3.0:1784949394.869619:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dd6c0379. 00000400:00000200:1.0:1784949394.869625:0:337118:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:0.0:1784949394.869626:0:337121:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000467d4d37. 00000020:00000010:3.0:1784949394.869633:0:342756:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000089ec28da. 02000000:00000001:0.0:1784949394.869635:0:337121:0:(sec.c:1884:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949394.869642:0:337118:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000020:00000010:3.0:1784949394.869646:0:342756:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000a4e0290. 00000400:00000010:0.0:1784949394.869649:0:337121:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 0000000005ecef72. 00000400:00000200:1.0:1784949394.869654:0:337118:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000020:00000010:3.0:1784949394.869660:0:342756:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000048e1370. 00000400:00000010:0.0:1784949394.869661:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ef38e22e. 00000400:00000200:1.0:1784949394.869664:0:337118:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000100:00000200:0.0:1784949394.869674:0:337121:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1871651289694720, portal 10 00000400:00000200:1.0:1784949394.869719:0:337118:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3996:3996:234:4294967295] 192.168.202.137@tcp LPNI seq info [2544:2544:8:4294967295] 00000100:00000040:3.0:1784949394.869720:0:342756:0:(service.c:1432:ptlrpc_at_set_timer()) armed mgs at +6s 00000100:00000001:0.0:1784949394.869721:0:337121:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1784949394.869725:0:337121:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628529469562752 : -115544239988864 : ffff96e9c29e9f80) 00000100:00000001:3.0:1784949394.869731:0:342756:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:3.0:1784949394.869737:0:342756:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000040:0.0:1784949394.869741:0:337121:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949394.869744:0:342756:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949394.869762:0:342756:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949394.869764:0:337118:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000100:00000001:0.0:1784949394.869765:0:337121:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949394.869776:0:337121:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949394.869780:0:337121:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000100:00000001:3.0:1784949394.869784:0:342756:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1784949394.869787:0:337121:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1871651289694720, offset 0 00000800:00000200:1.0:1784949394.869787:0:337118:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:1.0:1784949394.869802:0:337118:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000af953512. 00000100:00000001:3.0:1784949394.869802:0:342756:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000400:00000200:0.0:1784949394.869803:0:337121:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949394.869809:0:342756:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:3.0:1784949394.869820:0:342756:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 844 00000400:00000200:0.0:1784949394.869825:0:337121:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.137@tcp local destination 00000100:00000001:2.0:1784949394.869829:0:342757:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000800:00000200:1.0:1784949394.869830:0:337118:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000040:3.0:1784949394.869831:0:342756:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 00000000b1518b45 : new rpc_count 1 00000400:00000200:0.0:1784949394.869834:0:337121:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1784949394.869836:0:342757:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00000001:3.0:1784949394.869838:0:342756:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628534782163024 : -115538927388592 : ffff96eaff467850) 00000100:00000001:2.0:1784949394.869843:0:342757:0:(service.c:2244:ptlrpc_server_request_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949394.869845:0:337121:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000100:00000001:2.0:1784949394.869850:0:342757:0:(service.c:2483:ptlrpc_server_handle_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949394.869851:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@000000009bc35acb x1871651289694464/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:300/0 lens 224/0 e 0 to 0 dl 1784949405 ref 1 fl New:/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949394.869857:0:337121:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.137@tcp lpni_is_preferred = 0 00000800:00000200:1.0:1784949394.869859:0:337118:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000400:00000200:0.0:1784949394.869869:0:337121:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.137@tcp 00000800:00000200:1.0:1784949394.869873:0:337118:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000af953512 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1784949394.869883:0:337121:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3997:3997:233:4294967295] 192.168.202.137@tcp LPNI seq info [2545:2545:7:4294967295] 00000100:00000001:3.0:1784949394.869894:0:342756:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:3.0:1784949394.869900:0:342756:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 00000000dd6c0379. 00000400:00000200:0.0:1784949394.869905:0:337121:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949394.869915:0:337121:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000100:00000001:3.0:1784949394.869941:0:342756:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949394.869945:0:337118:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1784949394.869945:0:342756:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 00000000dd6c0379. 00000800:00000010:0.0:1784949394.869948:0:337121:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000034aacccc. 00000100:00100000:3.0:1784949394.869957:0:342756:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@000000009bc35acb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289694464:12345-0@lo:400: 00000100:00000040:1.0:1784949394.869959:0:337118:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: using a timeout of 0 usecs (0 jiffies) 00000800:00000200:0.0:1784949394.869964:0:337121:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000100:00000001:1.0:1784949394.869972:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306334252 00000100:00000200:3.0:1784949394.869975:0:342756:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289694464 00000800:00000200:0.0:1784949394.869983:0:337121:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000020:00000001:3.0:1784949394.869984:0:342756:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000100:00000001:1.0:1784949394.869987:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 0, simply extend its deadline from 4306334251 00000800:00000200:0.0:1784949394.869990:0:337121:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 0000000034aacccc type 1, nob 320 niov 1 nkiov 1 00000020:00000001:3.0:1784949394.869992:0:342756:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:3.0:1784949394.869998:0:342756:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949394.870001:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306334252 00000020:00000001:3.0:1784949394.870004:0:342756:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000100:00000001:0.0:1784949394.870008:0:337121:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949394.870009:0:342756:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072669120704 : -1040430912 : ffffffffc1fc48c0) 00000100:00000001:1.0:1784949394.870012:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 1, simply extend its deadline from 4306334251 00000020:00000001:3.0:1784949394.870019:0:342756:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000100:00000040:0.0:1784949394.870021:0:337121:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: using a timeout of 2266 usecs (3 jiffies) 00000020:00000001:3.0:1784949394.870023:0:342756:0:(tgt_handler.c:563:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949394.870025:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306334252 00000100:00000001:0.0:1784949394.870033:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 0 is 4306334213 00000020:00000001:3.0:1784949394.870033:0:342756:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000100:00000001:1.0:1784949394.870037:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 2, simply extend its deadline from 4306334251 00000020:00000001:3.0:1784949394.870039:0:342756:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000100:00000001:0.0:1784949394.870040:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 0, keep current deadline 4306334252 00000020:00000001:3.0:1784949394.870045:0:342756:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.870049:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 1 is 4306334213 00000020:00000001:3.0:1784949394.870051:0:342756:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:1.0:1784949394.870054:0:337118:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306334252 00000800:00000200:2.0:1784949394.870056:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1784949394.870058:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 1, keep current deadline 4306334252 00000100:00000001:3.0:1784949394.870058:0:342756:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 00000800:00000010:2.0:1784949394.870066:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000af953512. 02000000:00000001:3.0:1784949394.870069:0:342756:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 00000100:00000001:0.0:1784949394.870070:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 2 is 4306334213 00000100:00000001:1.0:1784949394.870070:0:337118:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-MDT0000: PM QoS request active for cpu 3, simply extend its deadline from 4306334251 00000400:00000200:2.0:1784949394.870074:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:3.0:1784949394.870079:0:342756:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000068fddc2b. 00000100:00000001:0.0:1784949394.870080:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 2, keep current deadline 4306334252 00000100:00000001:1.0:1784949394.870088:0:337118:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1784949394.870088:0:342756:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.870089:0:337121:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS new deadline estimation for cpu 3 is 4306334213 00000400:00000200:2.0:1784949394.870092:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:3.0:1784949394.870097:0:342756:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.870100:0:337121:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-lwp-MDT0000: PM QoS request active for cpu 3, keep current deadline 4306334252 00000100:00000001:1.0:1784949394.870103:0:337118:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949394.870107:0:342756:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1784949394.870108:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0c4f329 00000100:00000001:0.0:1784949394.870109:0:337121:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1784949394.870114:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1784949394.870115:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0c4f329. 00010000:00000040:3.0:1784949394.870118:0:342756:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289694464 00000100:00000001:0.0:1784949394.870119:0:337121:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949394.870125:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.870126:0:337114:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:3.0:1784949394.870129:0:342756:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00000100:00000001:0.0:1784949394.870130:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1784949394.870136:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000b83ea808 x1871651289694592/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949419 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949394.870138:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1784949394.870139:0:337118:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000200:3.0:1784949394.870142:0:342756:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@000000009bc35acb x1871651289694464/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:300/0 lens 224/224 e 0 to 0 dl 1784949405 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949394.870146:0:337118:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1784949394.870151:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949394.870158:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1784949394.870160:0:337118:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949394.870167:0:337118:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949394.870169:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1784949394.870175:0:337118:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.870176:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1784949394.870181:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949394.870187:0:337118:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.870192:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.870195:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00010000:00000001:3.0:1784949394.870197:0:342756:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:3.0:1784949394.870202:0:342756:0:(ldlm_lib.c:3350:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1784949394.870205:0:342756:0:(import.c:2149:obd_at_measure()) add 1 to 00000000fc2fb499 time=116 v=5 (1 1 1 1) 00000100:00000040:2.0:1784949394.870206:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@00000000b83ea808 x1871651289694592/t0(0) o8->lustre-OST0000-osc-MDT0000@192.168.202.137@tcp:28/4 lens 520/544 e 0 to 0 dl 1784949419 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1784949394.870217:0:342756:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000001:0.0:1784949394.870220:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949394.870225:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1784949394.870229:0:342756:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000008c235c88 refcount 6 to 0@lo 00000100:00000001:2.0:1784949394.870231:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1784949394.870235:0:342756:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534805621056 : -115538903930560 : ffff96eb00ac6940) 00000800:00000001:0.0:1784949394.870236:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1784949394.870242:0:342756:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 00000800:00000001:0.0:1784949394.870243:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 02000000:00000001:3.0:1784949394.870247:0:342756:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1784949394.870252:0:342756:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1784949394.870257:0:342756:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-0@lo 00000400:00000010:3.0:1784949394.870261:0:342756:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000026d65ad4. 00000100:00000200:3.0:1784949394.870266:0:342756:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 25, xid 1871651289694464, offset 224 00000400:00000200:3.0:1784949394.870273:0:342756:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:3.0:1784949394.870292:0:342756:0:(lib-move.c:4652:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:3.0:1784949394.870301:0:342756:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 25 MB=0x6a641c6440100 00000400:00000200:3.0:1784949394.870308:0:342756:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 224/224 into md 0x9931 [1] + 224 00000400:00000200:3.0:1784949394.870314:0:342756:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949394.870318:0:342756:0:(lib-msg.c:895:lnet_health_check()) health check: 0@lo->0@lo: PUT: OK 00000100:00000001:3.0:1784949394.870323:0:342756:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1784949394.870327:0:342756:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1784949394.870340:0:342756:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1784949394.870349:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949394.870354:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000034aacccc. 00000400:00000200:2.0:1784949394.870358:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1784949394.870359:0:342756:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1784949394.870368:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:3.0:1784949394.870370:0:342756:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000026d65ad4 00000400:00000010:3.0:1784949394.870375:0:342756:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000026d65ad4. 00000400:00000200:2.0:1784949394.870380:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000100:00000001:3.0:1784949394.870380:0:342756:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:3.0:1784949394.870383:0:342756:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:1.0:1784949394.870386:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000010:2.0:1784949394.870386:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000100:00000001:3.0:1784949394.870391:0:342756:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1784949394.870397:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.870397:0:337114:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1784949394.870398:0:342756:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000008c235c88 refcount 5 to 0@lo 00000800:00000001:1.0:1784949394.870406:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1784949394.870408:0:342756:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000100:00000200:2.0:1784949394.870409:0:337114:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1784949394.870414:0:342756:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1784949394.870419:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000800:00000001:1.0:1784949394.870419:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000020:00000001:3.0:1784949394.870424:0:342756:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000800:00000001:1.0:1784949394.870430:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1784949394.870436:0:342756:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009bc35acb x1871651289694464/t0(0) o400->1d45fc09-7805-4bc2-ab31-cf64613dd99c@0@lo:300/0 lens 224/224 e 0 to 0 dl 1784949405 ref 1 fl Interpret:/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1784949394.870450:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 520 into portal 28 MB=0x6a641c6440180 00000100:00000001:2.0:1784949394.870452:0:337114:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1784949394.870458:0:337112:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000100:00000040:2.0:1784949394.870463:0:337114:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1784949394.870480:0:342756:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@000000009bc35acb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:1d45fc09-7805-4bc2-ab31-cf64613dd99c+12:337119:x1871651289694464:12345-0@lo:400: Request processed in 577us (1304us total) trans 0 rc 0/0 00000400:00000100:0.0:1784949394.870486:0:337113:0:(lib-move.c:4357:lnet_parse_put()) Dropping PUT from 12345-192.168.202.137@tcp portal 28 match 1871651289694592 offset 0 length 520: 4 00000400:00000200:1.0:1784949394.870501:0:337112:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 12 MB=0x6a641c6440200 00000400:00000200:0.0:1784949394.870506:0:337113:0:(lib-msg.c:1116:lnet_is_health_check()) Msg 00000000169545f3 is in inconsistent state, don't perform health checking (-2, 0) 00000400:00000200:0.0:1784949394.870519:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 00000100:00000001:2.0:1784949394.870524:0:337114:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1784949394.870530:0:337112:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index c from 12345-192.168.202.137@tcp of length 224/224 into md 0x16bd [64] + 184720 00000100:00100000:3.0:1784949394.870531:0:342756:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 844 00000100:00000001:2.0:1784949394.870532:0:337114:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1784949394.870535:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000040:3.0:1784949394.870542:0:342756:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 00000000b1518b45 : new rpc_count 0 00000100:00000001:3.0:1784949394.870547:0:342756:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000800:00000001:1.0:1784949394.870552:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:3.0:1784949394.870552:0:342756:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000800:00000001:0.0:1784949394.870553:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.870558:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:3.0:1784949394.870569:0:342756:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000a4e0290. 00000100:00000001:2.0:1784949394.870570:0:337119:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000001:0.0:1784949394.870572:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000400:00000001:2.0:1784949394.870579:0:337119:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:3.0:1784949394.870581:0:342756:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000048e1370. 00000800:00000001:0.0:1784949394.870583:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1784949394.870584:0:337119:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000001:1.0:1784949394.870586:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1784949394.870591:0:337119:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 00000000603fb577. 00000020:00000010:3.0:1784949394.870593:0:342756:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000089ec28da. 00000400:00000200:1.0:1784949394.870596:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949394.870601:0:337119:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000020:00000040:3.0:1784949394.870603:0:342756:0:(genops.c:900:class_export_put()) PUTting export 00000000b1518b45 : new refcount 11 00000400:00000010:2.0:1784949394.870606:0:337119:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:1.0:1784949394.870614:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 02000000:00000001:3.0:1784949394.870615:0:342756:0:(sec.c:2643:sptlrpc_svc_free_rs()) Process entered 00000100:00000001:2.0:1784949394.870616:0:337119:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:3.0:1784949394.870622:0:342756:0:(sec_null.c:330:null_free_rs()) kvfree 'rs': 520 at 0000000068fddc2b. 00000100:00000200:2.0:1784949394.870627:0:337119:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1784949394.870636:0:337112:0:(events.c:286:request_in_callback()) Process entered 02000000:00000001:3.0:1784949394.870642:0:342756:0:(sec.c:2656:sptlrpc_svc_free_rs()) Process leaving 00000100:00000200:1.0:1784949394.870645:0:337112:0:(events.c:296:request_in_callback()) event type 2, status 0, service mdt 00000100:00000001:3.0:1784949394.870648:0:342756:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1784949394.870659:0:342756:0:(niobuf.c:1191:ptlrpc_register_rqbd()) mgs: registering portal 26 00000100:00000010:1.0:1784949394.870663:0:337112:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ced6fdca. 00000100:00000200:2.0:1784949394.870663:0:337119:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1784949394.870667:0:342756:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ea556a29. 00000100:00000040:1.0:1784949394.870675:0:337112:0:(events.c:337:request_in_callback()) incoming req@00000000ced6fdca x1871651289694720 msgsize 224 00000400:00000010:3.0:1784949394.870706:0:342756:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c9198cec. 00000100:00000001:2.0:1784949394.870713:0:337119:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1784949394.870718:0:337119:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.870724:0:337119:0:(client.c:1580:after_reply()) Process entered 00000100:00100000:1.0:1784949394.870731:0:337112:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.137@tcp (source: 12345-192.168.202.137@tcp) 02000000:00000001:2.0:1784949394.870732:0:337119:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1784949394.870736:0:337119:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1784949394.870740:0:337119:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1784949394.870742:0:337119:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1784949394.870747:0:337119:0:(import.c:2149:obd_at_measure()) add 5 to 00000000da7bd1d5 time=116 v=5 (5 5 5 5) 00000100:00001000:2.0:1784949394.870751:0:337119:0:(import.c:2149:obd_at_measure()) add 1 to 000000004ef2b0fe time=116 v=5 (1 1 1 1) 00000100:00000001:2.0:1784949394.870754:0:337119:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1784949394.870755:0:337119:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1784949394.870760:0:337112:0:(events.c:373:request_in_callback()) Process leaving 00010000:00000001:2.0:1784949394.870760:0:337119:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1784949394.870766:0:337119:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.870770:0:337119:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949394.870775:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000800:00000001:1.0:1784949394.870777:0:337112:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000100:00000001:2.0:1784949394.870787:0:337119:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000800:00000001:1.0:1784949394.870788:0:337112:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.870793:0:337119:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1784949394.870797:0:337119:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1784949394.870801:0:337119:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949394.870802:0:342763:0:(service.c:2282:ptlrpc_server_handle_req_in()) Process entered 00000100:00100000:0.0:1784949394.870808:0:342763:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1871651289694720 02000000:00000001:0.0:1784949394.870813:0:342763:0:(sec.c:2498:sptlrpc_svc_unwrap_request()) Process entered 00000100:00100000:2.0:1784949394.870818:0:337119:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@00000000aedb1960 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1d45fc09-7805-4bc2-ab31-cf64613dd99c:337119:1871651289694464:0@lo:400: 00000100:00000001:0.0:1784949394.870819:0:342763:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949394.870824:0:342763:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949394.870831:0:342763:0:(sec.c:2547:sptlrpc_svc_unwrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1784949394.870831:0:337119:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.870839:0:337119:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1784949394.870841:0:342763:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1871651289694720 00000100:00000040:2.0:1784949394.870846:0:337119:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@00000000aedb1960 x1871651289694464/t0(0) o400->MGC192.168.202.137@tcp@0@lo:26/25 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1784949394.870847:0:342763:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1784949394.870852:0:342763:0:(genops.c:825:class_conn2export()) looking for export cookie 0xeec19c0908bad615 00000020:00000001:0.0:1784949394.870856:0:342763:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1784949394.870861:0:342763:0:(lustre_handles.c:147:class_handle2object()) GET export 0000000084a824b3 refcount=5 00000100:00000001:2.0:1784949394.870865:0:337119:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1784949394.870867:0:342763:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000001:2.0:1784949394.870869:0:337119:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:0.0:1784949394.870872:0:342763:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446628533812660224 : -115539896891392 : ffff96eac57d0800) 02000000:00000010:2.0:1784949394.870874:0:337119:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000c405a384. 00000100:00000001:0.0:1784949394.870879:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 02000000:00000001:2.0:1784949394.870883:0:337119:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:0.0:1784949394.870886:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000020:00000001:2.0:1784949394.870887:0:337119:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:0.0:1784949394.870893:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1784949394.870894:0:337119:0:(genops.c:1244:class_import_put()) import 0000000036c7f24e refcount=3 obd=MGC192.168.202.137@tcp 00000020:00000001:2.0:1784949394.870931:0:337119:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1784949394.870934:0:337119:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 0000000036e798e7. 02000000:00000001:2.0:1784949394.870937:0:337119:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1784949394.870939:0:337119:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1784949394.870940:0:337119:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000aedb1960. 00000100:00000001:2.0:1784949394.870944:0:337119:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1784949394.870946:0:337119:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1784949394.870948:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00000001:2.0:1784949394.870948:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1784949394.870953:0:337119:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1784949394.870957:0:337119:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949394.870959:0:342763:0:(lu_object.c:1772:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005cc3cd5d. 00000100:00000001:2.0:1784949394.870960:0:337119:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1784949394.870965:0:337119:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1784949394.870968:0:342763:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cadcd9d2. 00000020:00000010:0.0:1784949394.870975:0:342763:0:(tgt_main.c:758:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000075e36e9f. 00000100:00000040:0.0:1784949394.870986:0:342763:0:(service.c:1432:ptlrpc_at_set_timer()) armed mdt at +6s 00000100:00000001:0.0:1784949394.870989:0:342763:0:(service.c:2003:ptlrpc_server_request_add()) Process entered 00000100:00000001:0.0:1784949394.870991:0:342763:0:(service.c:1919:ptlrpc_server_hpreq_init()) Process entered 00000100:00000001:0.0:1784949394.870993:0:342763:0:(service.c:1949:ptlrpc_server_hpreq_init()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.870999:0:342763:0:(nrs.c:878:ptlrpc_nrs_hpreq_add_nolock()) Process entered 00000100:00000001:0.0:1784949394.871001:0:342763:0:(nrs.c:886:ptlrpc_nrs_hpreq_add_nolock()) Process leaving 00000100:00000001:0.0:1784949394.871003:0:342763:0:(service.c:2101:ptlrpc_server_request_add()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.871013:0:342763:0:(service.c:2445:ptlrpc_server_handle_req_in()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.871022:0:342763:0:(service.c:2479:ptlrpc_server_handle_request()) Process entered 00000100:00000001:0.0:1784949394.871023:0:342763:0:(service.c:2223:ptlrpc_server_request_get()) Process entered 00000100:00100000:0.0:1784949394.871028:0:342763:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.137@tcp, seq: 820 00000100:00000040:0.0:1784949394.871030:0:342763:0:(service.c:2255:ptlrpc_server_request_get()) RPC GETting export 0000000084a824b3 : new rpc_count 1 00000100:00000001:0.0:1784949394.871032:0:342763:0:(service.c:2257:ptlrpc_server_request_get()) Process leaving (rc=18446628529554884352 : -115544154667264 : ffff96e9c7b48700) 00000100:00000040:0.0:1784949394.871048:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Interpret req@00000000ced6fdca x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:300/0 lens 224/0 e 0 to 0 dl 1784949405 ref 1 fl New:H/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949394.871057:0:342763:0:(service.c:1267:ptlrpc_update_export_timer()) Process entered 00000020:00000010:0.0:1784949394.871059:0:342763:0:(genops.c:1076:obd_export_timed_init()) kmalloced '(*data)': 48 at 000000006f6320b4. 00000100:00000001:0.0:1784949394.871061:0:342763:0:(service.c:1289:ptlrpc_update_export_timer()) Process leaving via err (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1784949394.871063:0:342763:0:(genops.c:1084:obd_export_timed_fini()) kfreed '*data': 48 at 000000006f6320b4. 00000100:00100000:0.0:1784949394.871067:0:342763:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@00000000ced6fdca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289694720:12345-192.168.202.137@tcp:400: 00000100:00000200:0.0:1784949394.871071:0:342763:0:(service.c:2563:ptlrpc_server_handle_request()) got req 1871651289694720 00000020:00000001:0.0:1784949394.871073:0:342763:0:(tgt_handler.c:764:tgt_request_handle()) Process entered 00000020:00000001:0.0:1784949394.871076:0:342763:0:(tgt_handler.c:683:process_req_last_xid()) Process entered 00000020:00000001:0.0:1784949394.871078:0:342763:0:(tgt_handler.c:748:process_req_last_xid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949394.871080:0:342763:0:(tgt_handler.c:620:tgt_handler_find_check()) Process entered 00000020:00000001:0.0:1784949394.871082:0:342763:0:(tgt_handler.c:657:tgt_handler_find_check()) Process leaving (rc=18446744072661359136 : -1048192480 : ffffffffc185da20) 00000020:00000001:0.0:1784949394.871085:0:342763:0:(tgt_handler.c:550:tgt_handle_recovery()) Process entered 00000020:00000001:0.0:1784949394.871088:0:342763:0:(tgt_handler.c:607:tgt_handle_recovery()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1784949394.871090:0:342763:0:(tgt_handler.c:375:tgt_handle_request0()) Process entered 00000020:00000001:0.0:1784949394.871092:0:342763:0:(tgt_handler.c:296:tgt_request_preprocess()) Process entered 00000020:00000001:0.0:1784949394.871094:0:342763:0:(tgt_handler.c:359:tgt_request_preprocess()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949394.871097:0:342763:0:(tgt_handler.c:1160:tgt_obd_ping()) Process entered 00000100:00000001:0.0:1784949394.871100:0:342763:0:(pack_generic.c:296:lustre_pack_reply_v2()) Process entered 02000000:00000001:0.0:1784949394.871102:0:342763:0:(sec.c:2569:sptlrpc_svc_alloc_rs()) Process entered 02000000:00000010:0.0:1784949394.871106:0:342763:0:(sec_null.c:305:null_alloc_rs()) kmalloced '(rs)': 520 at 0000000077ad1613. 02000000:00000001:0.0:1784949394.871108:0:342763:0:(sec.c:2606:sptlrpc_svc_alloc_rs()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.871110:0:342763:0:(pack_generic.c:331:lustre_pack_reply_v2()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949394.871112:0:342763:0:(tgt_handler.c:1181:tgt_obd_ping()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1784949394.871115:0:342763:0:(ldlm_lib.c:3328:target_committed_to_req()) last_committed 0, transno 0, xid 1871651289694720 00010000:00000001:0.0:1784949394.871118:0:342763:0:(ldlm_lib.c:3397:target_send_reply()) Process entered 00010000:00000200:0.0:1784949394.871122:0:342763:0:(ldlm_lib.c:3385:target_send_reply_msg()) @@@ sending reply req@00000000ced6fdca x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:300/0 lens 224/224 e 0 to 0 dl 1784949405 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949394.871129:0:342763:0:(ldlm_lib.c:3340:target_pack_pool_reply()) Process entered 00010000:00000001:0.0:1784949394.871131:0:342763:0:(ldlm_lib.c:3361:target_pack_pool_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949394.871133:0:342763:0:(import.c:2149:obd_at_measure()) add 1 to 00000000bb3011d5 time=111 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949394.871136:0:342763:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1784949394.871139:0:342763:0:(connection.c:161:ptlrpc_connection_addref()) conn=000000007c53774e refcount 9 to 192.168.202.137@tcp 00000100:00000001:0.0:1784949394.871141:0:342763:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628534823712320 : -115538885839296 : ffff96eb01c07640) 02000000:00000001:0.0:1784949394.871144:0:342763:0:(sec.c:2620:sptlrpc_svc_wrap_reply()) Process entered 02000000:00000001:0.0:1784949394.871145:0:342763:0:(sec.c:2631:sptlrpc_svc_wrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.871147:0:342763:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1784949394.871150:0:342763:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.137@tcp 00000400:00000010:0.0:1784949394.871152:0:342763:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b525ec8a. 00000100:00000200:0.0:1784949394.871155:0:342763:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 10, xid 1871651289694720, offset 224 00000400:00000200:0.0:1784949394.871159:0:342763:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.137@tcp 00000400:00000200:0.0:1784949394.871166:0:342763:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:0.0:1784949394.871171:0:342763:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3998:3998:234:4294967295] 192.168.202.137@tcp LPNI seq info [2546:2546:8:4294967295] 00000400:00000200:0.0:1784949394.871180:0:342763:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : PUT try# 0 00000800:00000200:0.0:1784949394.871185:0:342763:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:0.0:1784949394.871188:0:342763:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009ff40a64. 00000800:00000200:0.0:1784949394.871192:0:342763:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:0.0:1784949394.871197:0:342763:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:0.0:1784949394.871200:0:342763:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 000000009ff40a64 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1784949394.871213:0:342763:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949394.871217:0:342763:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=000000007c53774e refcount 8 to 192.168.202.137@tcp 00010000:00000001:0.0:1784949394.871219:0:342763:0:(ldlm_lib.c:3409:target_send_reply()) Process leaving 00000020:00000001:0.0:1784949394.871220:0:342763:0:(tgt_handler.c:497:tgt_handle_request0()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1784949394.871222:0:342763:0:(tgt_handler.c:905:tgt_request_handle()) Process leaving 00000100:00000040:0.0:1784949394.871225:0:342763:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000ced6fdca x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000_UUID@192.168.202.137@tcp:300/0 lens 224/224 e 0 to 0 dl 1784949405 ref 1 fl Interpret:H/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949394.871235:0:342763:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@00000000ced6fdca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:337121:x1871651289694720:12345-192.168.202.137@tcp:400: Request processed in 176us (559us total) trans 0 rc 0/0 00000100:00100000:0.0:1784949394.871243:0:342763:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.137@tcp, seq: 820 00000100:00000040:0.0:1784949394.871246:0:342763:0:(service.c:1113:ptlrpc_server_finish_active_request()) RPC PUTting export 0000000084a824b3 : new rpc_count 0 00000100:00000001:0.0:1784949394.871248:0:342763:0:(service.c:1955:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:0.0:1784949394.871251:0:342763:0:(service.c:1966:ptlrpc_server_hpreq_fini()) Process leaving 00000020:00000010:0.0:1784949394.871254:0:342763:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cadcd9d2. 00000020:00000010:0.0:1784949394.871258:0:342763:0:(tgt_main.c:770:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000075e36e9f. 00000020:00000010:0.0:1784949394.871268:0:342763:0:(lu_object.c:1700:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005cc3cd5d. 00000020:00000040:0.0:1784949394.871274:0:342763:0:(genops.c:900:class_export_put()) PUTting export 0000000084a824b3 : new refcount 4 00000100:00000001:0.0:1784949394.871277:0:342763:0:(service.c:2635:ptlrpc_server_handle_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1784949394.871307:0:337114:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1784949394.871315:0:337114:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009ff40a64. 00000400:00000200:2.0:1784949394.871320:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949394.871325:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000400:00000200:2.0:1784949394.871328:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b525ec8a 00000400:00000010:2.0:1784949394.871330:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b525ec8a. 00000100:00000001:2.0:1784949394.871333:0:337114:0:(events.c:386:reply_out_callback()) Process entered 00000100:00000001:2.0:1784949394.871335:0:337114:0:(events.c:398:reply_out_callback()) Process leaving 00000800:00000001:2.0:1784949394.871339:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949394.871345:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949394.871347:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949394.871349:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949394.871353:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : PUT - for me 00000400:00000200:2.0:1784949394.871361:0:337114:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 224 into portal 10 MB=0x6a641c6440200 00000400:00000200:2.0:1784949394.871367:0:337114:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.137@tcp of length 224/224 into md 0x9941 [1] + 224 00000800:00000001:2.0:1784949394.871372:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949394.871377:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949394.871379:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949394.871383:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: PUT: OK 00000100:00000001:2.0:1784949394.871386:0:337114:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1784949394.871390:0:337114:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1784949394.871400:0:337114:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1784949394.871418:0:337114:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1784949394.871423:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949394.871425:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.871467:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949394.871482:0:337121:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1784949394.871487:0:337121:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1784949394.871490:0:337121:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1784949394.871493:0:337121:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at 0000000005ecef72. 00000400:00000200:0.0:1784949394.871500:0:337121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ef38e22e 00000400:00000010:0.0:1784949394.871503:0:337121:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ef38e22e. 00000100:00000001:0.0:1784949394.871507:0:337121:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1784949394.871514:0:337121:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1784949394.871528:0:337121:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949394.871537:0:337121:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1784949394.871538:0:337121:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.871558:0:337121:0:(client.c:1580:after_reply()) Process entered 02000000:00000001:0.0:1784949394.871570:0:337121:0:(sec.c:1135:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1784949394.871572:0:337121:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1784949394.871574:0:337121:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1784949394.871576:0:337121:0:(sec.c:1191:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1784949394.871582:0:337121:0:(import.c:2149:obd_at_measure()) add 5 to 0000000094f34b47 time=111 v=5 (5 5 5 5) 00000100:00001000:0.0:1784949394.871585:0:337121:0:(import.c:2149:obd_at_measure()) add 1 to 000000003a1ec4b3 time=111 v=5 (1 1 1 1) 00000100:00000001:0.0:1784949394.871588:0:337121:0:(client.c:1482:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1784949394.871590:0:337121:0:(client.c:1520:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1784949394.871592:0:337121:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1784949394.871595:0:337121:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=184649) req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1784949394.871604:0:337121:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.871607:0:337121:0:(client.c:3096:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1784949394.871608:0:337121:0:(client.c:3108:ptlrpc_free_committed()) lustre-MDT0000-lwp-MDT0000: skip recheck: last_committed 0 00000100:00000001:0.0:1784949394.871610:0:337121:0:(client.c:3109:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1784949394.871613:0:337121:0:(client.c:1784:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1784949394.871617:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949394.871624:0:337121:0:(client.c:2341:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1784949394.871626:0:337121:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1784949394.871628:0:337121:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1784949394.871631:0:337121:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1784949394.871638:0:337121:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@000000003bf8e8ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:337121:1871651289694720:192.168.202.137@tcp:400: 00000100:00000001:0.0:1784949394.871643:0:337121:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.871645:0:337121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1784949394.871647:0:337121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@000000003bf8e8ad x1871651289694720/t0(0) o400->lustre-MDT0000-lwp-MDT0000@192.168.202.137@tcp:12/10 lens 224/224 e 0 to 0 dl 1784949410 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1784949394.871654:0:337121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1784949394.871655:0:337121:0:(sec.c:1899:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1784949394.871657:0:337121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000467d4d37. 02000000:00000001:0.0:1784949394.871661:0:337121:0:(sec.c:1915:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1784949394.871663:0:337121:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1784949394.871664:0:337121:0:(genops.c:1244:class_import_put()) import 000000006c89a515 refcount=2 obd=lustre-MDT0000-lwp-MDT0000 00000020:00000001:0.0:1784949394.871667:0:337121:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1784949394.871669:0:337121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at 000000002a753d9d. 02000000:00000001:0.0:1784949394.871671:0:337121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1784949394.871673:0:337121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1784949394.871674:0:337121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000003bf8e8ad. 00000100:00000001:0.0:1784949394.871702:0:337121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1784949394.871704:0:337121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.871706:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1784949394.871729:0:337121:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1784949394.871731:0:337121:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1784949394.871733:0:337121:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1784949394.871734:0:337121:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1784949394.912974:0:343945:0:(connection.c:75:cpu_latency_work()) work item of 000000000b5607a2 (cpu 2) has reached its deadline 4306334252, at 4306334253 00000100:00000040:1.0:1784949394.912988:0:343964:0:(connection.c:75:cpu_latency_work()) work item of 00000000358ef526 (cpu 1) has reached its deadline 4306334252, at 4306334253 00000100:00000010:2.0:1784949394.912992:0:343945:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f99e6ef2. 00000100:00000040:3.0:1784949394.912994:0:344064:0:(connection.c:75:cpu_latency_work()) work item of 0000000089f3c7ed (cpu 3) has reached its deadline 4306334252, at 4306334253 00000100:00000010:1.0:1784949394.913001:0:343964:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b9b6859c. 00000100:00000010:3.0:1784949394.913007:0:344064:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f59314ac. 00000100:00000040:0.0:1784949394.913051:0:343997:0:(connection.c:75:cpu_latency_work()) work item of 0000000094d761ac (cpu 0) has reached its deadline 4306334252, at 4306334253 00000100:00000010:0.0:1784949394.913062:0:343997:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000882090e1. 00010000:00000001:2.0:1784949395.700042:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949395.700052:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949395.700060:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949395.700062:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949395.700139:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949395.700146:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949395.700151:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949396.724029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949396.724038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949396.724047:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949396.724048:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949396.724173:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949396.724179:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949396.724184:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1784949397.748029:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949397.748038:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949397.748046:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949397.748048:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1784949397.748161:0:342751:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1784949397.748167:0:342751:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1784949397.748172:0:342751:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1784949398.452145:0:344095:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.13:988 00000400:00000100:3.0:1784949398.452152:0:344096:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.30:988 00000400:00000100:0.0:1784949398.452153:0:344094:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.11:988 00000400:00000100:0.0:1784949398.452278:0:344094:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.11@tcp at host 111.222.173.11:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:1.0:1784949398.452281:0:344095:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.13@tcp at host 111.222.173.13:988 took too long: that node may be hung or experiencing high load. 00000400:00000100:2.0:1784949398.452281:0:344097:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.9:988 00000400:00000100:3.0:1784949398.452314:0:344096:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.30@tcp at host 111.222.173.30:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949398.452315:0:344094:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.11@tcp: down 00000400:00000200:1.0:1784949398.452316:0:344095:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.13@tcp: down 00000400:00000200:1.0:1784949398.452337:0:344095:0:(router.c:1888:lnet_notify()) Old news 00000400:00000100:2.0:1784949398.452341:0:344097:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.9@tcp at host 111.222.173.9:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949398.452349:0:344094:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1784949398.452359:0:344095:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.13@tcp 00000400:00000200:2.0:1784949398.452366:0:344097:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.9@tcp: down 00000400:00000200:3.0:1784949398.452369:0:344096:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.30@tcp: down 00000800:00000010:1.0:1784949398.452374:0:344095:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000002ddfcc2d. 00000400:00000200:2.0:1784949398.452381:0:344097:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949398.452394:0:344095:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000800:00000100:2.0:1784949398.452395:0:344097:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.9@tcp 00000800:00000010:2.0:1784949398.452409:0:344097:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000001ee120ca. 00000800:00000200:3.0:1784949398.452415:0:344096:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 111.222.173.30@tcp down 00000400:00000200:2.0:1784949398.452423:0:344097:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949398.452424:0:344095:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.13@tcp: GET: NETWORK_TIMEOUT 00000400:00000100:0.0:1784949398.452427:0:344057:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.3:988 00000400:00000200:2.0:1784949398.452442:0:344097:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.9@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949398.452450:0:344095:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 0 next ping: 11671 health :900 00000400:00000200:1.0:1784949398.452463:0:344095:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1784949398.452465:0:344097:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949398.452474:0:344095:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.13@tcp: 1 00000400:00000200:2.0:1784949398.452479:0:344097:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.9@tcp: 1 00000400:00000200:1.0:1784949398.452493:0:344095:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000100:3.0:1784949398.452494:0:344099:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.15:988 00000400:00000100:0.0:1784949398.452505:0:344057:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.3@tcp at host 111.222.173.3:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:0.0:1784949398.452517:0:344057:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.3@tcp: down 00000400:00000200:2.0:1784949398.452529:0:344097:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000200:0.0:1784949398.452540:0:344057:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1784949398.452552:0:344057:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.3@tcp 00000800:00000010:0.0:1784949398.452557:0:344057:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000000475ac2. 00000400:00000200:0.0:1784949398.452568:0:344057:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:0.0:1784949398.452578:0:344057:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.3@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:0.0:1784949398.452587:0:344057:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949398.452595:0:344057:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.3@tcp: 1 00000400:00000200:1.0:1784949398.452598:0:337117:0:(peer.c:4145:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000100:2.0:1784949398.452605:0:344100:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.7:988 00000400:00000200:0.0:1784949398.452608:0:344057:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000100:3.0:1784949398.452612:0:344099:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.15@tcp at host 111.222.173.15:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:1.0:1784949398.452636:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x861 00000400:00000200:3.0:1784949398.452683:0:344099:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.15@tcp: down 00000400:00000100:2.0:1784949398.452691:0:344100:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.7@tcp at host 111.222.173.7:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:3.0:1784949398.452693:0:344099:0:(router.c:1888:lnet_notify()) Old news 00000400:00000001:1.0:1784949398.452703:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1784949398.452712:0:344100:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.7@tcp: down 00000400:00000001:1.0:1784949398.452715:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000100:3.0:1784949398.452716:0:344099:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.15@tcp 00000400:00000100:0.0:1784949398.452720:0:344098:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.5:988 00000400:00000200:2.0:1784949398.452722:0:344100:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949398.452725:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b039801a 00000400:00000010:1.0:1784949398.452737:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000b039801a. 00000800:00000100:2.0:1784949398.452737:0:344100:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.7@tcp 00000800:00000010:3.0:1784949398.452747:0:344099:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000600b3b47. 00000800:00000010:2.0:1784949398.452751:0:344100:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000b9b32818. 00000400:00000200:1.0:1784949398.452755:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949398.452765:0:344100:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949398.452775:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.13@tcp: -114 00000400:00000200:3.0:1784949398.452779:0:344099:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000100:0.0:1784949398.452780:0:344098:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.5@tcp at host 111.222.173.5:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949398.452781:0:344100:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.7@tcp: GET: NETWORK_TIMEOUT 00000400:00000010:1.0:1784949398.452791:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000001df3b40e. 00000400:00000200:0.0:1784949398.452804:0:344098:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.5@tcp: down 00000400:00000200:2.0:1784949398.452808:0:344100:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949398.452812:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.13@tcp:-110 00000400:00000200:0.0:1784949398.452820:0:344098:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:3.0:1784949398.452824:0:344099:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.15@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:2.0:1784949398.452825:0:344100:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.7@tcp: 1 00000400:00000200:1.0:1784949398.452830:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.13@tcp(00000000d09edccd) state 0x2061 rc -110 00000800:00000100:0.0:1784949398.452840:0:344098:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.5@tcp 00000400:00000200:2.0:1784949398.452844:0:344100:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000800:00000010:0.0:1784949398.452854:0:344098:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000009e7fe55b. 00000400:00000200:0.0:1784949398.452862:0:344098:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000200:1.0:1784949398.452870:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.13@tcp 00000400:00000200:0.0:1784949398.452872:0:344098:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.5@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:3.0:1784949398.452895:0:344099:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949398.452899:0:344098:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1784949398.452959:0:344098:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.5@tcp: 1 00000400:00000200:1.0:1784949398.452961:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x861 00000400:00000200:0.0:1784949398.452970:0:344098:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000001:1.0:1784949398.452975:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000100:2.0:1784949398.452976:0:344046:0:(lib-socket.c:438:lnet_sock_connect()) Error -110 connecting 1023 -> 111.222.173.1:988 00000400:00000200:3.0:1784949398.452983:0:344099:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.15@tcp: 1 00000400:00000001:1.0:1784949398.452984:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949398.452992:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e0ff87dd 00000400:00000010:1.0:1784949398.452997:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000e0ff87dd. 00000400:00000200:1.0:1784949398.453011:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:3.0:1784949398.453014:0:344099:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000100:2.0:1784949398.453021:0:344046:0:(acceptor.c:121:lnet_connect_console_error()) Connection to 111.222.173.1@tcp at host 111.222.173.1:988 took too long: that node may be hung or experiencing high load. 00000400:00000200:2.0:1784949398.453030:0:344046:0:(router.c:1791:lnet_notify()) 192.168.202.137@tcp notifying 111.222.173.1@tcp: down 00000400:00000200:1.0:1784949398.453053:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.9@tcp: -114 00000400:00000010:1.0:1784949398.453061:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000fff5e130. 00000400:00000200:2.0:1784949398.453072:0:344046:0:(router.c:1888:lnet_notify()) Old news 00000400:00000200:1.0:1784949398.453073:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.9@tcp:-110 00000400:00000200:1.0:1784949398.453091:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.9@tcp(00000000f676b11b) state 0x2061 rc -110 00000800:00000100:2.0:1784949398.453098:0:344046:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 2 len 0 0@<0:0>->111.222.173.1@tcp 00000400:00000200:1.0:1784949398.453108:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.9@tcp 00000800:00000010:2.0:1784949398.453112:0:344046:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 0000000005678171. 00000400:00000200:1.0:1784949398.453123:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x861 00000400:00000200:2.0:1784949398.453125:0:344046:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -110, hstatus = 10 00000400:00000001:1.0:1784949398.453135:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949398.453144:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1784949398.453146:0:344046:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->111.222.173.1@tcp: GET: NETWORK_TIMEOUT 00000400:00000200:1.0:1784949398.453151:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005c5c5c3 00000400:00000010:1.0:1784949398.453159:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000005c5c5c3. 00000400:00000200:1.0:1784949398.453166:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:2.0:1784949398.453169:0:344046:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1784949398.453177:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.3@tcp: -114 00000400:00000200:2.0:1784949398.453183:0:344046:0:(peer.c:3156:lnet_discovery_event_send()) Ping Send to 111.222.173.1@tcp: 1 00000400:00000010:1.0:1784949398.453188:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000461737cb. 00000400:00000200:1.0:1784949398.453204:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.3@tcp:-110 00000400:00000200:2.0:1784949398.453205:0:344046:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000200:1.0:1784949398.453221:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.3@tcp(00000000333a109b) state 0x2061 rc -110 00000400:00000200:1.0:1784949398.453232:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.3@tcp 00000400:00000200:1.0:1784949398.453241:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x861 00000400:00000001:1.0:1784949398.453246:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949398.453249:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949398.453252:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000066cfac65 00000400:00000010:1.0:1784949398.453255:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000066cfac65. 00000400:00000200:1.0:1784949398.453261:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949398.453266:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.7@tcp: -114 00000400:00000010:1.0:1784949398.453271:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000b105ef1a. 00000400:00000200:1.0:1784949398.453278:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.7@tcp:-110 00000400:00000200:1.0:1784949398.453283:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.7@tcp(000000009f6806e4) state 0x2061 rc -110 00000400:00000200:1.0:1784949398.453288:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.7@tcp 00000400:00000200:1.0:1784949398.453293:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x861 00000400:00000001:1.0:1784949398.453297:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949398.453300:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949398.453302:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000600895c9 00000400:00000010:1.0:1784949398.453305:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000600895c9. 00000400:00000200:1.0:1784949398.453309:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949398.453314:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.5@tcp: -114 00000400:00000010:1.0:1784949398.453317:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000194fc0c0. 00000400:00000200:1.0:1784949398.453325:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.5@tcp:-110 00000400:00000200:1.0:1784949398.453329:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.5@tcp(000000005cfa6fce) state 0x2061 rc -110 00000400:00000200:1.0:1784949398.453334:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.5@tcp 00000400:00000200:1.0:1784949398.453340:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x861 00000400:00000001:1.0:1784949398.453344:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949398.453347:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949398.453350:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a0df5e09 00000400:00000010:1.0:1784949398.453354:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000a0df5e09. 00000400:00000200:1.0:1784949398.453357:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949398.453361:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.15@tcp: -114 00000400:00000010:1.0:1784949398.453365:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000004041a48e. 00000400:00000200:1.0:1784949398.453373:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.15@tcp:-110 00000400:00000200:1.0:1784949398.453378:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.15@tcp(0000000002302dbf) state 0x2061 rc -110 00000400:00000200:1.0:1784949398.453383:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.15@tcp 00000400:00000200:1.0:1784949398.453388:0:337117:0:(peer.c:4257:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x861 00000400:00000001:1.0:1784949398.453393:0:337117:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1784949398.453395:0:337117:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1784949398.453398:0:337117:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000004238acf 00000400:00000010:1.0:1784949398.453402:0:337117:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 0000000004238acf. 00000400:00000200:1.0:1784949398.453405:0:337117:0:(peer.c:3204:lnet_discovery_event_handler()) Received event: 6 00000400:00000200:1.0:1784949398.453409:0:337117:0:(peer.c:2528:lnet_peer_queue_for_discovery()) Queue peer 111.222.173.1@tcp: -114 00000400:00000010:1.0:1784949398.453414:0:337117:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 0000000032fe57fd. 00000400:00000200:1.0:1784949398.453420:0:337117:0:(peer.c:3891:lnet_peer_ping_failed()) peer 111.222.173.1@tcp:-110 00000400:00000200:1.0:1784949398.453424:0:337117:0:(peer.c:4275:lnet_peer_discovery()) peer 111.222.173.1@tcp(00000000513db5ee) state 0x2061 rc -110 00000400:00000200:1.0:1784949398.453429:0:337117:0:(peer.c:2544:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 111.222.173.1@tcp 00010000:00000001:2.0:1784949398.771995:0:343945:0:(ldlm_pool.c:319:ldlm_srv_pool_recalc()) Process entered 00010000:00000001:2.0:1784949398.772002:0:343945:0:(ldlm_pool.c:351:ldlm_srv_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1784949398.772007:0:343945:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1784949398.772008:0:343945:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1784949398.772078:0:342750:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1784949398.772083:0:342750:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1784949398.772086:0:342750:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:2.0:1784949399.156030:0:337116:0:(lib-move.c:3448:lnet_recover_local_nis()) attempting to recover local ni: 192.168.202.137@tcp 00000400:00000010:2.0:1784949399.156041:0:337116:0:(lib-move.c:3455:lnet_recover_local_nis()) alloc '(ev_info)': 24 at 00000000b4b4f5a0. 00000400:00000010:2.0:1784949399.156055:0:337116:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000c405a384. 00000400:00000010:2.0:1784949399.156062:0:337116:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ab3a6503. 00000400:00000010:2.0:1784949399.156076:0:337116:0:(lib-lnet.h:443:lnet_rspt_alloc()) alloc 'rspt': 64 at 000000008f1e4bb5. 00000400:00000200:2.0:1784949399.156086:0:337116:0:(lib-move.c:5397:LNetGet()) LNetGet -> 12345-192.168.202.137@tcp 00000400:00000200:2.0:1784949399.156107:0:337116:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to NMR: 192.168.202.137@tcp local destination 00000400:00000200:2.0:1784949399.156118:0:337116:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.137@tcp 00000400:00000200:2.0:1784949399.156126:0:337116:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [3999:3999:241:4294967295] 192.168.202.137@tcp LPNI seq info [2547:2547:8:4294967295] 00000400:00000200:2.0:1784949399.156139:0:337116:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.137@tcp 00000400:00000200:2.0:1784949399.156154:0:337116:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) : GET try# 0 00000800:00000200:2.0:1784949399.156165:0:337116:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.137@tcp 00000800:00000010:2.0:1784949399.156174:0:337116:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000aba78344. 00000800:00000200:2.0:1784949399.156185:0:337116:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:2.0:1784949399.156195:0:337116:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:988 00000800:00000200:2.0:1784949399.156201:0:337116:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000aba78344 type 2, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1784949399.156241:0:337116:0:(lib-msg.c:534:lnet_ni_add_to_recoveryq_locked()) 192.168.202.137@tcp added to recovery queue. ping count: 1 next ping: 11673 health :300 00000800:00000200:1.0:1784949399.156398:0:337112:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1784949399.156404:0:337112:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000aba78344. 00000400:00000200:1.0:1784949399.156409:0:337112:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1784949399.156416:0:337112:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: GET: OK 00000400:00000200:1.0:1784949399.156422:0:337112:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 5 status: 0 00000400:00000200:1.0:1784949399.156425:0:337112:0:(lib-move.c:4135:lnet_mt_event_handler()) 192.168.202.137@tcp recovery message sent successfully:0 00000800:00000001:3.0:1784949399.156432:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949399.156446:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1784949399.156450:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949399.156457:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1784949399.156464:0:337113:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : GET - for me 00000400:00000200:3.0:1784949399.156476:0:337113:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.202.137@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:3.0:1784949399.156484:0:337113:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.202.137@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:3.0:1784949399.156493:0:337113:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.202.137@tcp to MR: 192.168.202.137@tcp local destination 00000400:00000200:3.0:1784949399.156496:0:337113:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.137@tcp NI seq info: [4000:4000:241:4294967295] 192.168.202.137@tcp LPNI seq info [2548:2548:8:4294967295] 00000400:00000200:3.0:1784949399.156503:0:337113:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) -> 192.168.202.137@tcp(192.168.202.137@tcp:192.168.202.137@tcp) 192.168.202.137@tcp : REPLY try# 0 00000800:00000200:3.0:1784949399.156507:0:337113:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.137@tcp 00000800:00000010:3.0:1784949399.156510:0:337113:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000c9dda688. 00000800:00000200:3.0:1784949399.156514:0:337113:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000a62815f8] -> 12345-192.168.202.137@tcp (6) 00000800:00000200:3.0:1784949399.156518:0:337113:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.137@tcp ip 192.168.202.137:1023 00000800:00000200:3.0:1784949399.156520:0:337113:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet 00000000c9dda688 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:3.0:1784949399.156575:0:337113:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1784949399.156577:0:337113:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000c9dda688. 00000400:00000200:3.0:1784949399.156582:0:337113:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1784949399.156585:0:337113:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000800:00000001:3.0:1784949399.156596:0:337113:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1784949399.156599:0:337113:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949399.156603:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949399.156614:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1784949399.156617:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949399.156620:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949399.156626:0:337114:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.202.137@tcp(192.168.202.137@tcp) <- 192.168.202.137@tcp : REPLY - for me 00000400:00000200:2.0:1784949399.156637:0:337114:0:(lib-move.c:4486:lnet_parse_reply()) 192.168.202.137@tcp: Reply from 12345-192.168.202.137@tcp of length 48/48 into md 0x9955 00000800:00000001:2.0:1784949399.156659:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949399.156664:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1784949399.156666:0:337114:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1784949399.156669:0:337114:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.137@tcp->192.168.202.137@tcp: REPLY: OK 00000400:00000200:2.0:1784949399.156674:0:337114:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ab3a6503 00000400:00000010:2.0:1784949399.156676:0:337114:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at 00000000ab3a6503. 00000400:00000200:2.0:1784949399.156680:0:337114:0:(lib-move.c:4120:lnet_mt_event_handler()) Received event: 3 status: 0 00000400:00000010:2.0:1784949399.156683:0:337114:0:(lib-move.c:4143:lnet_mt_event_handler()) kfreed 'ev_info': 24 at 00000000b4b4f5a0. 00000400:00000010:2.0:1784949399.156686:0:337114:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000c405a384. 00000800:00000001:2.0:1784949399.156692:0:337114:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1784949399.156694:0:337114:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) Debug log: 130974 lines, 130974 kept, 0 dropped, 0 bad.